// import com.alibaba.druid.DbType_ptr;
// import com.alibaba.druid.sql.ast.SQLObject;
// import com.alibaba.druid.sql.ast.expr.SQLNumberExpr;
// import com.alibaba.druid.sql.dialect.mysql.parser.MySqlLexer;
// import com.alibaba.druid.util.FnvHash;
// import com.alibaba.druid.util.StringUtils;

// import java.math.BigDecimal;
// import java.math.BigInteger;
// import java.util.ArrayList;
// import java.util.Arrays;
// import java.util.List;
// import java.util.time_t;

// import static com.alibaba.druid.sql.parser.CharTypes.*;
// import static com.alibaba.druid.sql.parser.LayoutCharacters.EOI;
// import static com.alibaba.druid.sql.parser.SQLParserFeature.*;
// import static com.alibaba.druid.sql.parser.Token_ptr(new Token(&Token::*;

#include "Lexer.h"
#include <climits>
#include <algorithm>
#include "SymbolTable.h"
#include "Keywords.h"
#include "LayoutCharacters.h"
#include "SQLParserFeature.h"
#include "CharTypes.h"
#include "SQLType.h"
#include "NotAllowCommentException.h"
#include "SQLParserUtils.h"
#include "Token.h"
#include "ParserException.h"
#include "../../DbType.h"
#include "../ast/SQLObject.h"
#include "../ast/expr/SQLNumberExpr.h"
#include "../dialect/mysql/parser/MySqlLexer.h"
#include "../../utils/FnvHash.h"
#include "../../utils/StringUtils.h"
#include "../../utils/log.h"
#include "../../Exception/IllegalStateException.h"
#include "../../Exception/NumberFormatException.h"

SymbolTable_ptr Lexer::symbols_l2 = SymbolTable_ptr(new SymbolTable(512));
long Lexer::MULTMIN_RADIX_TEN = LONG_MIN / 10;
long Lexer::N_MULTMAX_RADIX_TEN = -LONG_MAX / 10;
int Lexer::digits[11] = {0};

BOOL_ptr Lexer::__init = Lexer::init();
BOOL_ptr Lexer::init()
{
  for (int i = '0'; i <= '9'; ++i)
  {
    Lexer::digits[i] = i - '0';
  }

  return BOOL::TRUE;
}

Lexer::Lexer(std::wstring input)
    : Lexer(input, BOOL::TRUE)
{
  this->commentHandler = commentHandler;
}

Lexer::Lexer(std::wstring input, Lexer_CommentHandler_ptr commentHandler)
    : Lexer(input, BOOL::TRUE)
{
  this->commentHandler = commentHandler;
}

Lexer::Lexer(std::wstring input, BOOL_ptr skipComment)
{
  this->skipComment = skipComment;

  this->text = input;
  this->pos_ = 0;
  ch = charAt(pos_);
  while (ch == 0x200B || ch == L'\n')
  {
    if (ch == L'\n')
    {
      line++;
    }
    ch = charAt(++pos_);
  }
  buf = NULL;
}

Lexer::Lexer(wchar_t *input, int inputLength, BOOL_ptr skipComment)
    : Lexer(std::wstring(input), skipComment)
{
}

Lexer::Lexer(std::wstring input, Lexer_CommentHandler_ptr commentHandler, DbType_ptr dbType)
    : Lexer(input, BOOL::TRUE)
{

  this->commentHandler = commentHandler;
  this->dbType = dbType;

  if (DbType::sqlite.name->c_str() == dbType->name->c_str())
  {
    this->keywords = Keywords::SQLITE_KEYWORDS;
  }
  else if (DbType::dm.name->c_str() == dbType->name->c_str())
  {
    this->keywords = Keywords::DM_KEYWORDS;
  }
}

Lexer::~Lexer()
{
  if (buf != NULL)
  {
    delete[] buf;
    buf = NULL;
  }
}

BOOL_ptr Lexer::isKeepComments()
{
  return keepComments;
}

void Lexer::setKeepComments(BOOL_ptr keepComments)
{
  this->keepComments = keepComments;
}

Lexer_CommentHandler_ptr Lexer::getCommentHandler()
{
  return commentHandler;
}

void Lexer::setCommentHandler(Lexer_CommentHandler_ptr commentHandler)
{
  this->commentHandler = commentHandler;
}

wchar_t Lexer::charAt(int index)
{
  if (index >= text.length())
  {
    return LayoutCharacters::EOI;
  }

  return text.at(index);
}

std::wstring Lexer::addSymbol()
{
  return subString(mark_, bufPos);
}

std::wstring Lexer::subString(int offset, int count)
{
  return text.substr(offset, count);
}

std::wstring Lexer::subString(int offset)
{
  return text.substr(offset);
}

wstring_ptr Lexer::sub_chars(int offset, int count)
{
  wchar_t chars[count];
  // text.getChars(offset, offset + count, chars, 0);
  for (int i = offset; i < offset + count; i++)
  {
    chars[i - offset] = text[i];
  }

  std::wstring tmp = chars;
  return make_wstring_ptr(tmp);
}

void Lexer::initBuff(int size)
{
  LOG_DEBUG << "in initBuff";
  if (buf == nullptr)
  {
    if (size < 32)
    {
      buf = new wchar_t[32];
    }
    else
    {
      buf = new wchar_t[size + 32];
    }
  }
  else if (wcslen(buf) < size)
  {
    // buf = Arrays.copyOf(buf, size);
  }
}

void Lexer::arraycopy(int srcPos, wchar_t *dest, int destPos, int length)
{
  // text.getChars(srcPos, srcPos + length, dest, destPos);
  for (int i = srcPos; i < srcPos + length; i++)
  {
    dest[i - length + destPos] = text[i];
  }
}

BOOL_ptr Lexer::isAllowComment()
{
  return allowComment;
}

void Lexer::setAllowComment(BOOL_ptr allowComment)
{
  this->allowComment = allowComment;
}

int Lexer::Lexer::nextVarIndex()
{
  return ++varIndex;
}

Keywords_ptr Lexer::getKeywords()
{
  return keywords;
}

// 出现多次调用mark()后调用reset()会有问题
//  @Deprecated
Lexer_SavePoint_ptr Lexer::mark()
{
  return this->savePoint = markOut();
}

Lexer_SavePoint_ptr Lexer::markOut()
{
  Lexer_SavePoint_ptr savePoint = Lexer_SavePoint_ptr(new Lexer_SavePoint());
  savePoint->bp = pos_;
  savePoint->sp = bufPos;
  savePoint->np = mark_;
  savePoint->ch = ch;
  savePoint->token_ = token_;
  savePoint->stringVal_ = stringVal_;
  savePoint->hash = hash;
  savePoint->hashLCase_ = hashLCase_;
  savePoint->startPos = startPos;
  return savePoint;
}

void Lexer::reset(Lexer_SavePoint_ptr savePoint)
{
  this->pos_ = savePoint->bp;
  this->bufPos = savePoint->sp;
  this->mark_ = savePoint->np;
  this->ch = savePoint->ch;
  this->token_ = savePoint->token_;
  this->stringVal_ = savePoint->stringVal_;
  this->hash = savePoint->hash;
  this->hashLCase_ = savePoint->hashLCase_;
  this->startPos = savePoint->startPos;
}

// 出现多次调用mark()后调用reset()会有问题
//  @Deprecated
void Lexer::reset()
{
  this->reset(this->savePoint);
}

void Lexer::reset(int pos_)
{
  this->pos_ = pos_;
  this->ch = charAt(pos_);
}

void Lexer::scanChar()
{
  ch = charAt(++pos_);
}

void Lexer::unscan()
{
  ch = charAt(--pos_);
}

bool Lexer::isEOF()
{
  return pos_ >= text.length();
}

/**
 * Report an error at the given position using the provided arguments.
 */
void Lexer::lexError(std::wstring key, Object_list_ptr args)
{
  token_ = Token_ptr(new Token(&Token::ERROR));
}

/**
 * Return the current token_, set by nextToken().
 */
Token_ptr Lexer::token()
{
  return token_;
}

void Lexer::setToken(Token_ptr token_)
{
  this->token_ = token_;
}

DbType_ptr Lexer::getDbType()
{
  return this->dbType;
}

std::wstring Lexer::info()
{
  int line = 1;
  int column = 1;
  for (int i = 0; i < startPos; ++i, column++)
  {
    wchar_t ch = text.at(i);
    if (ch == '\n')
    {
      column = 1;
      line++;
    }
  }

  this->posLine = line;
  this->posColumn = column;

  std::stringstream buf;

  buf << "pos_ ";
  buf << pos_;
  buf << ", line ";
  buf << line;
  buf << ", column ";
  buf << column;
  if (token_ != nullptr)
  {
    if (token_->name != nullptr)
    {
      buf << ", token_ ";
      buf << token_->name->c_str();
    }
    else
    {
      buf << ", token_ ";
      buf << token_->name->c_str();
    }
  }

  if (token_->name->c_str() == Token::IDENTIFIER.name->c_str() ||
      token_->name->c_str() == Token::LITERAL_ALIAS.name->c_str() ||
      token_->name->c_str() == Token::LITERAL_CHARS.name->c_str())
  {
    buf << " ";
    buf << stringVal()->c_str();
  }

  if (isEnabled(SQLParserFeature_ptr(new SQLParserFeature(&SQLParserFeature::PrintSQLWhileParsingFailed))))
  {
    buf << ", SQL : ";
    buf << encode_util::UnicodeToUTF8(text);
  }

  // LOG_INFO << buf.str();

  std::string s = buf.str();
  // buf >> s;
  LOG_INFO << s;
  std::wstring ws(s.begin(), s.end());
  // std::wcerr << L"====+++++++++++++++++++++++++++++++++================" << ws;
  return ws;
}

void Lexer::nextTokenComma()
{
  if (ch == L' ')
  {
    scanChar();
  }

  if (ch == L',' || ch == L'，')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::COMMA));
    return;
  }

  if (ch == L')' || ch == L'）')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::RPAREN));
    return;
  }

  if (ch == L'.')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::DOT));
    return;
  }

  if (ch == L'a' || ch == L'A')
  {
    wchar_t ch_next = charAt(pos_ + 1);
    if (ch_next == L's' || ch_next == L'S')
    {
      wchar_t ch_next_2 = charAt(pos_ + 2);
      if (ch_next_2 == L' ')
      {
        pos_ += 2;
        ch = L' ';
        token_ = Token_ptr(new Token(&Token::AS));
        stringVal_ = L"AS";
        return;
      }
    }
  }

  nextToken();
}

void Lexer::nextTokenCommaValue()
{
  if (ch == L' ')
  {
    scanChar();
  }

  if (ch == L',' || ch == L'，')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::COMMA));
    return;
  }

  if (ch == L')' || ch == L'）')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::RPAREN));
    return;
  }

  if (ch == L'.')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::DOT));
    return;
  }

  if (ch == L'a' || ch == L'A')
  {
    wchar_t ch_next = charAt(pos_ + 1);
    if (ch_next == L's' || ch_next == L'S')
    {
      wchar_t ch_next_2 = charAt(pos_ + 2);
      if (ch_next_2 == L' ')
      {
        pos_ += 2;
        ch = L' ';
        token_ = Token_ptr(new Token(&Token::AS));
        stringVal_ = L"AS";
        return;
      }
    }
  }

  nextTokenValue();
}

void Lexer::nextTokenEq()
{
  if (ch == L' ')
  {
    scanChar();
  }

  if (ch == L'=')
  {
    scanChar();
    if (ch == L'=')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::EQEQ));
    }
    else if (ch == L'>')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::EQGT));
    }
    token_ = Token_ptr(new Token(&Token::EQ));
    return;
  }

  if (ch == L'.')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::DOT));
    return;
  }

  if (ch == L'a' || ch == L'A')
  {
    wchar_t ch_next = charAt(pos_ + 1);
    if (ch_next == L's' || ch_next == L'S')
    {
      wchar_t ch_next_2 = charAt(pos_ + 2);
      if (ch_next_2 == L' ')
      {
        pos_ += 2;
        ch = L' ';
        token_ = Token_ptr(new Token(&Token::AS));
        stringVal_ = L"AS";
        return;
      }
    }
  }

  nextToken();
}

void Lexer::nextTokenLParen()
{
  if (ch == L' ')
  {
    scanChar();
  }

  if (ch == L'(' || ch == L'（')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::LPAREN));
    return;
  }
  nextToken();
}

void Lexer::nextTokenValue()
{
  LOG_INFO << "in nextTokenValue";
  this->startPos = pos_;
  WLOG_INFO << L"ch:[" << ch << "]" << std::endl;
  while (ch == L' ')
  {
    scanChar();
  }

  WLOG_INFO << L"ch:[" << ch << "]" << std::endl;
  if (ch == L'\'')
  {
    bufPos = 0;
    if (dbType->name->c_str() == DbType::mysql.name->c_str())
    {
      scanString2();
    }
    else
    {
      scanString();
    }
    return;
  }

  WLOG_INFO << L"ch:[" << ch << "]" << std::endl;
  if (ch == L'"' &&
      !isEnabled(SQLParserFeature_ptr(new SQLParserFeature(&SQLParserFeature::KeepNameQuotes))))
  {
    bufPos = 0;
    scanString2_d();
    return;
  }

  WLOG_INFO << L"ch:[" << ch << "]" << std::endl;
  if (ch == L'0')
  {
    bufPos = 0;
    if (charAt(pos_ + 1) == L'x')
    {
      scanChar();
      scanChar();
      scanHexaDecimal();
    }
    else
    {
      scanNumber();
    }
    return;
  }

  WLOG_INFO << L"ch:[" << ch << "]" << std::endl;
  if (ch > L'0' && ch <= L'9')
  {
    bufPos = 0;
    scanNumber();
    return;
  }

  WLOG_INFO << L"ch:[" << ch << "]" << std::endl;
  if (ch == L'-')
  {
    wchar_t next = charAt(pos_ + 1);
    if (next >= L'0' && next <= L'9')
    {
      bufPos = 0;
      scanNumber();
      return;
    }
  }

  WLOG_INFO << L"ch:[" << ch << "]" << std::endl;
  if (ch == L'?')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::QUES));
    return;
  }

  WLOG_INFO << L"ch:[" << ch << "]" << std::endl;
  if (ch == L'n' || ch == L'N')
  {
    wchar_t c1 = 0, c2, c3, c4;
    if (pos_ + 4 < text.length() &&
        ((c1 = text.at(pos_ + 1)) == L'u' || c1 == L'U') &&
        ((c2 = text.at(pos_ + 2)) == L'l' || c2 == L'L') &&
        ((c3 = text.at(pos_ + 3)) == L'l' || c3 == L'L') &&
        (CharTypes::isWhitespace(c4 = text.at(pos_ + 4)) || c4 == L',' || c4 == L')' || c4 == L'）'))
    {
      pos_ += 4;
      ch = c4;
      token_ = Token_ptr(new Token(&Token::XNULL));
      stringVal_ = L"NULL";
      return;
    }

    if (c1 == L'\'')
    {
      bufPos = 0;
      ++pos_;
      ch = L'\'';
      scanString();
      token_ = Token_ptr(new Token(&Token::LITERAL_NCHARS));
      return;
    }
  }

  WLOG_INFO << L"ch:[" << ch << "]" << std::endl;
  if (ch == L')' || ch == L'）')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::RPAREN));
    return;
  }

  WLOG_INFO << L"ch:[" << ch << "]" << std::endl;
  if (ch == L'（')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::LPAREN));
    return;
  }

  WLOG_INFO << L"ch:[" << ch << "]" << std::endl;
  if (ch == '$' && isVaraintChar(charAt(pos_ + 1)))
  {
    scanVariable();
    return;
  }

  WLOG_INFO << L"ch:[" << ch << "]" << std::endl;
  if (CharTypes::isFirstIdentifierChar(ch))
  {
    LOG_INFO << "goto scanIdentifier";
    scanIdentifier();
    return;
  }

  LOG_INFO << "goto nextToken";
  nextToken();
}

bool Lexer::isVaraintChar(wchar_t ch)
{
  return ch == L'{' || (ch >= L'0' && ch <= L'9');
}

void Lexer::nextTokenBy()
{
  while (ch == L' ')
  {
    scanChar();
  }

  if (ch == L'b' || ch == L'B')
  {
    wchar_t ch_next = charAt(pos_ + 1);
    if (ch_next == L'y' || ch_next == L'Y')
    {
      wchar_t ch_next_2 = charAt(pos_ + 2);
      if (ch_next_2 == L' ')
      {
        pos_ += 2;
        ch = L' ';
        token_ = Token_ptr(new Token(&Token::BY));
        stringVal_ = L"BY";
        return;
      }
    }
  }

  nextToken();
}

void Lexer::nextTokenNotOrNull()
{
  while (ch == L' ')
  {
    scanChar();
  }

  if ((ch == L'n' || ch == L'N') && pos_ + 3 < text.length())
  {
    wchar_t c1 = text.at(pos_ + 1);
    wchar_t c2 = text.at(pos_ + 2);
    wchar_t c3 = text.at(pos_ + 3);

    if ((c1 == L'o' || c1 == L'O') && (c2 == L't' || c2 == L'T') && CharTypes::isWhitespace(c3))
    {
      pos_ += 3;
      ch = c3;
      token_ = Token_ptr(new Token(&Token::NOT));
      stringVal_ = L"NOT";
      return;
    }

    wchar_t c4;
    if (pos_ + 4 < text.length() && (c1 == L'u' || c1 == L'U') && (c2 == L'l' || c2 == L'L') && (c3 == L'l' || c3 == L'L') && CharTypes::isWhitespace(c4 = text.at(pos_ + 4)))
    {
      pos_ += 4;
      ch = c4;
      token_ = Token_ptr(new Token(&Token::XNULL));
      stringVal_ = L"NULL";
      return;
    }
  }

  nextToken();
}

void Lexer::nextTokenIdent()
{
  while (ch == L' ')
  {
    scanChar();
  }

  if (ch == L'$' && isVaraintChar(charAt(pos_ + 1)))
  {
    scanVariable();
    return;
  }

  if (CharTypes::isFirstIdentifierChar(ch) && ch != L'（')
  {
    scanIdentifier();
    return;
  }

  if (ch == L')')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::RPAREN));
    return;
  }

  nextToken();
}

SQLType_ptr Lexer::scanSQLType()
{
_loop:
  for (;;)
  {
    while (CharTypes::isWhitespace(ch))
    {
      ch = charAt(++pos_);
    }

    if (ch == '/')
    {
      if (pos_ + 1 < text.length() && text.at(pos_ + 1) == '*')
      {
        int index = text.find(L"*/", pos_ + 2);
        if (index == std::wstring::npos)
        {
          return SQLType::UNKNOWN;
        }

        pos_ = index + 2;
        ch = charAt(pos_);
        continue;
      }
      else if (pos_ + 2 < text.length() && text.at(pos_ + 1) == L' ' && text.at(pos_ + 2) == L'*' && dbType->name->c_str() == DbType::odps.name->c_str())
      {
        int index = text.find(L"* /", pos_ + 3);
        if (index == std::wstring::npos)
        {
          return SQLType::UNKNOWN;
        }

        pos_ = index + 3;
        ch = charAt(pos_);
        continue;
      }
    }

    if (dbType->name->c_str() == DbType::odps.name->c_str())
    {
      while (ch == L';')
      {
        ch = charAt(++pos_);

        if (isEOF())
        {
          return SQLType::EMPTY;
        }

        while (CharTypes::isWhitespace(ch))
        {
          ch = charAt(++pos_);
        }
        goto _loop;
      }
    }

    if (pos_ + 1 < text.length() && ((ch == L'-' && text.at(pos_ + 1) == L'-') ||
                                     (ch == L'—' && text.at(pos_ + 1) == L'—')))
    {
      int index = text.find(L'\n', pos_ + 2);
      if (index == std::wstring::npos)
      {
        reset(0);
        nextToken();
        if (token_->name->c_str() == Token::XEOF.name->c_str())
        {
          return SQLType::EMPTY;
        }

        return SQLType::UNKNOWN;
      }

      pos_ = index + 1;
      ch = charAt(pos_);
      continue;
    }

    break;
  }

  while (ch == L'(')
  {
    ch = charAt(++pos_);

    while (CharTypes::isWhitespace(ch))
    {
      ch = charAt(++pos_);
    }
  }

  long hashCode = FnvHash::BASIC;
  for (;;)
  {
    wchar_t c;
    if (ch >= L'a' && ch <= L'z')
    {
      c = ch;
    }
    else if (ch >= L'A' && ch <= L'Z')
    {
      c = (wchar_t)(ch + 32);
    }
    else
    {
      break;
    }
    hashCode ^= c;
    hashCode *= FnvHash::PRIME;

    ch = charAt(++pos_);
  }

  if (ch == L'_' || (ch >= L'0' && ch <= L'9'))
  {
    return SQLType::UNKNOWN;
  }

  if (hashCode == FnvHash::Constants::SELECT)
  {
    return SQLType::SELECT;
  }
  else if (hashCode == FnvHash::Constants::INSERT)
  {
    return SQLType::INSERT;
  }
  else if (hashCode == FnvHash::Constants::DELETE)
  {
    return SQLType::DELETE;
  }
  else if (hashCode == FnvHash::Constants::UPDATE)
  {
    return SQLType::UPDATE;
  }
  else if (hashCode == FnvHash::Constants::REPLACE)
  {
    return SQLType::REPLACE;
  }
  else if (hashCode == FnvHash::Constants::TRUNCATE)
  {
    return SQLType::TRUNCATE;
  }
  else if (hashCode == FnvHash::Constants::MERGE)
  {
    return SQLType::MERGE;
  }
  else if (hashCode == FnvHash::Constants::CREATE)
  {
    return SQLType::CREATE;
  }
  else if (hashCode == FnvHash::Constants::ALTER)
  {
    return SQLType::ALTER;
  }
  else if (hashCode == FnvHash::Constants::SHOW)
  {
    nextToken();
    if (identifierEquals(FnvHash::Constants::STATISTIC))
    {
      return SQLType::SHOW_STATISTIC;
    }
    else if (identifierEquals(FnvHash::Constants::STATISTIC_LIST))
    {
      return SQLType::SHOW_STATISTIC_LIST;
    }
    else if (identifierEquals(FnvHash::Constants::TABLES) ||
             identifierEquals(L"TABLES；"))
    {
      return SQLType::SHOW_TABLES;
    }
    else if (identifierEquals(FnvHash::Constants::PARTITIONS))
    {
      return SQLType::SHOW_PARTITIONS;
    }
    else if (identifierEquals(FnvHash::Constants::CATALOGS))
    {
      return SQLType::SHOW_CATALOGS;
    }
    else if (identifierEquals(FnvHash::Constants::FUNCTIONS))
    {
      return SQLType::SHOW_FUNCTIONS;
    }
    else if (identifierEquals(FnvHash::Constants::ROLES))
    {
      return SQLType::SHOW_ROLES;
    }
    else if (identifierEquals(FnvHash::Constants::ROLE))
    {
      return SQLType::SHOW_ROLE;
    }
    else if (identifierEquals(FnvHash::Constants::LABEL))
    {
      return SQLType::SHOW_LABEL;
    }
    else if (identifierEquals(FnvHash::Constants::GRANTS))
    {
      return SQLType::SHOW_GRANTS;
    }
    else if (identifierEquals(FnvHash::Constants::GRANT) || token_->name->c_str() == Token::GRANT.name->c_str())
    {
      return SQLType::SHOW_GRANT;
    }
    else if (identifierEquals(FnvHash::Constants::RECYCLEBIN))
    {
      return SQLType::SHOW_RECYCLEBIN;
    }
    else if (identifierEquals(L"VARIABLES"))
    {
      return SQLType::SHOW_VARIABLES;
    }
    else if (identifierEquals(L"HISTORY"))
    {
      return SQLType::SHOW_HISTORY;
    }
    else if (identifierEquals(L"PACKAGES"))
    {
      return SQLType::SHOW_PACKAGES;
    }
    else if (identifierEquals(L"PACKAGE"))
    {
      return SQLType::SHOW_PACKAGE;
    }
    else if (identifierEquals(L"CHANGELOGS"))
    {
      return SQLType::SHOW_CHANGELOGS;
    }
    else if (identifierEquals(L"ACL"))
    {
      return SQLType::SHOW_ACL;
    }
    else if (token_->name->c_str() == Token::CREATE.name->c_str())
    {
      nextToken();
      if (token_->name->c_str() == Token::TABLE.name->c_str())
      {
        return SQLType::SHOW_CREATE_TABLE;
      }
    }
    return SQLType::SHOW;
  }
  else if (hashCode == FnvHash::Constants::DESC)
  {
    return SQLType::DESC;
  }
  else if (hashCode == FnvHash::Constants::DESCRIBE)
  {
    return SQLType::DESC;
  }
  else if (hashCode == FnvHash::Constants::SET)
  {
    nextToken();
    if (identifierEquals(FnvHash::Constants::LABEL))
    {
      return SQLType::SET_LABEL;
    }
    else if (identifierEquals(L"PROJECT"))
    {
      return SQLType::SET_PROJECT;
    }
    return SQLType::SET;
  }
  else if (hashCode == FnvHash::Constants::KILL)
  {
    return SQLType::KILL;
  }
  else if (hashCode == FnvHash::Constants::MSCK)
  {
    return SQLType::MSCK;
  }
  else if (hashCode == FnvHash::Constants::USE)
  {
    return SQLType::USE;
  }
  else if (hashCode == FnvHash::Constants::DROP)
  {
    return SQLType::DROP;
  }
  else if (hashCode == FnvHash::Constants::LIST)
  {
    nextToken();
    if (identifierEquals(FnvHash::Constants::USERS))
    {
      return SQLType::LIST_USERS;
    }
    else if (identifierEquals(FnvHash::Constants::TABLES))
    {
      return SQLType::LIST_TABLES;
    }
    else if (identifierEquals(FnvHash::Constants::ROLES))
    {
      return SQLType::LIST_ROLES;
    }
    else if (identifierEquals(FnvHash::Constants::TEMPORARY))
    {
      return SQLType::LIST_TEMPORARY_OUTPUT;
    }
    else if (identifierEquals(L"TENANT"))
    {
      nextToken();
      if (identifierEquals(FnvHash::Constants::ROLES))
      {
        return SQLType::LIST_TENANT_ROLES;
      }
    }
    else if (identifierEquals(L"TRUSTEDPROJECTS"))
    {
      return SQLType::LIST_TRUSTEDPROJECTS;
    }
    else if (identifierEquals(L"ACCOUNTPROVIDERS"))
    {
      return SQLType::LIST_ACCOUNTPROVIDERS;
    }
    return SQLType::LIST;
  }
  else if (hashCode == FnvHash::Constants::ROLLBACK)
  {
    return SQLType::ROLLBACK;
  }
  else if (hashCode == FnvHash::Constants::COMMIT)
  {
    return SQLType::COMMIT;
  }
  else if (hashCode == FnvHash::Constants::WHO)
  {
    return SQLType::WHO;
  }
  else if (hashCode == FnvHash::Constants::GRANT)
  {
    return SQLType::GRANT;
  }
  else if (hashCode == FnvHash::Constants::REVOKE)
  {
    return SQLType::REVOKE;
  }
  else if (hashCode == FnvHash::Constants::ANALYZE)
  {
    return SQLType::ANALYZE;
  }
  else if (hashCode == FnvHash::Constants::EXPLAIN)
  {
    return SQLType::EXPLAIN;
  }
  else if (hashCode == FnvHash::Constants::READ)
  {
    return SQLType::READ;
  }
  else if (hashCode == FnvHash::Constants::WITH)
  {
    return SQLType::WITH;
  }
  else if (hashCode == FnvHash::Constants::DUMP)
  {
    nextToken();
    if (identifierEquals(FnvHash::Constants::DATA))
    {
      return SQLType::DUMP_DATA;
    }
  }
  else if (hashCode == FnvHash::Constants::ADD)
  {
    nextToken();
    if (token_->name->c_str() == Token::USER.name->c_str() ||
        identifierEquals(FnvHash::Constants::USER))
    {
      return SQLType::ADD_USER;
    }
    else if (token_->name->c_str() == Token::TABLE.name->c_str())
    {
      return SQLType::ADD_TABLE;
    }
    else if (token_->name->c_str() == Token::FUNCTION.name->c_str())
    {
      return SQLType::ADD_FUNCTION;
    }
    else if (identifierEquals(FnvHash::Constants::STATISTIC))
    {
      return SQLType::ADD_STATISTIC;
    }
    else if (identifierEquals(FnvHash::Constants::RESOURCE))
    {
      return SQLType::ADD_RESOURCE;
    }
    else if (identifierEquals(L"VOLUME"))
    {
      return SQLType::ADD_VOLUME;
    }
    else if (identifierEquals(L"ACCOUNTPROVIDER"))
    {
      return SQLType::ADD_ACCOUNTPROVIDER;
    }
    else if (identifierEquals(L"TRUSTEDPROJECT"))
    {
      return SQLType::ADD_TRUSTEDPROJECT;
    }
    else
    {
      return SQLType::ADD;
    }
  }
  else if (hashCode == FnvHash::Constants::REMOVE)
  {
    nextToken();
    if (token_->name->c_str() == Token::USER.name->c_str() ||
        identifierEquals(FnvHash::Constants::USER))
    {
      return SQLType::REMOVE_USER;
    }

    if (identifierEquals(FnvHash::Constants::RESOURCE))
    {
      return SQLType::REMOVE_RESOURCE;
    }
    return SQLType::REMOVE;
  }
  else if (hashCode == FnvHash::Constants::TUNNEL)
  {
    nextToken();
    if (identifierEquals(FnvHash::Constants::DOWNLOAD))
    {
      return SQLType::TUNNEL_DOWNLOAD;
    }
  }
  else if (hashCode == FnvHash::Constants::UPLOAD)
  {
    return SQLType::UPLOAD;
  }
  else if (hashCode == FnvHash::Constants::WHOAMI)
  {
    return SQLType::WHOAMI;
  }
  else if (hashCode == FnvHash::Constants::COUNT)
  {
    return SQLType::COUNT;
  }
  else if (hashCode == FnvHash::Constants::CLONE)
  {
    return SQLType::CLONE;
  }
  else if (hashCode == FnvHash::Constants::LOAD)
  {
    return SQLType::LOAD;
  }
  else if (hashCode == FnvHash::Constants::INSTALL)
  {
    return SQLType::INSTALL;
  }
  else if (hashCode == FnvHash::Constants::UNLOAD)
  {
    return SQLType::UNLOAD;
  }
  else if (hashCode == FnvHash::Constants::ALLOW)
  {
    return SQLType::ALLOW;
  }
  else if (hashCode == FnvHash::Constants::PURGE)
  {
    return SQLType::PURGE;
  }
  else if (hashCode == FnvHash::Constants::RESTORE)
  {
    return SQLType::RESTORE;
  }
  else if (hashCode == FnvHash::Constants::EXSTORE)
  {
    return SQLType::EXSTORE;
  }
  else if (hashCode == FnvHash::Constants::UNDO)
  {
    return SQLType::UNDO;
  }
  else if (hashCode == FnvHash::Constants::REMOVE)
  {
    return SQLType::REMOVE;
  }
  else if (hashCode == FnvHash::Constants::FROM)
  {
    if (dbType->name->c_str() == DbType::odps.name->c_str() ||
        dbType->name->c_str() == DbType::hive.name->c_str())
    {
      return SQLType::INSERT_MULTI;
    }
  }
  else if (hashCode == FnvHash::Constants::ADD)
  {
    return SQLType::ADD;
  }
  else if (hashCode == FnvHash::Constants::IF)
  {
    return SQLType::SCRIPT;
  }
  else if (hashCode == FnvHash::Constants::FUNCTION)
  {
    if (dbType->name->c_str() == DbType::odps.name->c_str())
    {
      return SQLType::SCRIPT;
    }
  }
  else if (hashCode == FnvHash::Constants::BEGIN)
  {
    if (dbType->name->c_str() == DbType::odps.name->c_str() ||
        dbType->name->c_str() == DbType::oracle.name->c_str())
    {
      return SQLType::SCRIPT;
    }
  }
  else if (ch == L'@')
  {
    nextToken();
    if (token_->name->c_str() == Token::VARIANT.name->c_str() &&
        dbType->name->c_str() == DbType::odps.name->c_str())
    {
      nextToken();

      if (token_->name->c_str() == Token::TABLE.name->c_str())
      {
        return SQLType::SCRIPT;
      }

      // datatype
      if (token_->name->c_str() == Token::IDENTIFIER.name->c_str())
      {
        nextToken();
      }

      if (token_->name->c_str() == Token::COLONEQ.name->c_str() ||
          token_->name->c_str() == Token::SEMI.name->c_str())
      {
        return SQLType::SCRIPT;
      }
    }
  }

  if (ch == LayoutCharacters::EOI)
  {
    return SQLType::EMPTY;
  }

  return SQLType::UNKNOWN;
}

SQLType_ptr Lexer::scanSQLTypeV2()
{
  SQLType_ptr sqlType = scanSQLType();
  if (sqlType == SQLType::CREATE)
  {
    nextToken();
    if (token_->name->c_str() == Token::USER.name->c_str() ||
        identifierEquals(FnvHash::Constants::USER))
    {
      return SQLType::CREATE_USER;
    }

  _for:
    for (int i = 0; i < 1000; ++i)
    {
      if (token_->name->c_str() == Token::XEOF.name->c_str() ||
          token_->name->c_str() == Token::ERROR.name->c_str())
      {
        goto _for;
      }
      else if (token_->name->c_str() == Token::TABLE.name->c_str())
      {
        sqlType = SQLType::CREATE_TABLE;
      }
      else if (token_->name->c_str() == Token::VIEW.name->c_str())
      {
        if (sqlType == SQLType::CREATE)
        {
          sqlType = SQLType::CREATE_VIEW;
          goto _for;
        }
      }
      else if (token_->name->c_str() == Token::FUNCTION.name->c_str())
      {
        if (sqlType == SQLType::CREATE)
        {
          sqlType = SQLType::CREATE_FUNCTION;
          goto _for;
        }
      }
      else if (token_->name->c_str() == Token::SELECT.name->c_str())
      {
        if (sqlType == SQLType::CREATE_TABLE)
        {
          sqlType = SQLType::CREATE_TABLE_AS_SELECT;
          goto _for;
        }
      }
      else
      {
        if (sqlType->name->c_str() == SQLType::CREATE->name->c_str() &&
            identifierEquals(FnvHash::Constants::ROLE))
        {
          sqlType = SQLType::CREATE_ROLE;
          goto _for;
        }
        if (sqlType->name->c_str() == SQLType::CREATE->name->c_str() &&
            identifierEquals(FnvHash::Constants::PACKAGE))
        {
          sqlType = SQLType::CREATE_PACKAGE;
          goto _for;
        }
      }

      nextToken();
    }
  }
  else if (sqlType == SQLType::DROP)
  {
    nextToken();
    if (token_->name->c_str() == Token::USER.name->c_str() ||
        identifierEquals(FnvHash::Constants::USER))
    {
      return SQLType::DROP_USER;
    }
    else if (token_->name->c_str() == Token::TABLE.name->c_str())
    {
      return SQLType::DROP_TABLE;
    }
    else if (token_->name->c_str() == Token::VIEW.name->c_str())
    {
      return SQLType::DROP_VIEW;
    }
    else if (token_->name->c_str() == Token::FUNCTION.name->c_str())
    {
      return SQLType::DROP_FUNCTION;
    }
    else if (identifierEquals(FnvHash::Constants::ROLE))
    {
      return SQLType::DROP_ROLE;
    }
    else if (identifierEquals(FnvHash::Constants::RESOURCE))
    {
      return SQLType::DROP_RESOURCE;
    }
    else if (identifierEquals(FnvHash::Constants::MATERIALIZED))
    {
      nextToken();
      if (token_->name->c_str() == Token::VIEW.name->c_str())
      {
        return SQLType::DROP_MATERIALIZED_VIEW;
      }
    }
  }
  else if (sqlType->name->c_str() == SQLType::ALTER->name->c_str())
  {
    nextToken();
    if (token_->name->c_str() == Token::USER.name->c_str() ||
        identifierEquals(FnvHash::Constants::USER))
    {
      return SQLType::ALTER_USER;
    }
    else if (token_->name->c_str() == Token::TABLE.name->c_str())
    {
      return SQLType::ALTER_TABLE;
    }
    else if (token_->name->c_str() == Token::VIEW.name->c_str())
    {
      return SQLType::ALTER_VIEW;
    }
  }
  else if (sqlType == SQLType::INSERT)
  {
    nextToken();
    bool overwrite = token_->name->c_str() == Token::OVERWRITE.name->c_str();
    for (int i = 0; i < 1000; ++i)
    {
      nextToken();

      if (token_->name->c_str() == Token::SELECT.name->c_str())
      {
        return overwrite
                   ? SQLType::INSERT_OVERWRITE_SELECT
                   : SQLType::INSERT_INTO_SELECT;
      }
      else if (token_->name->c_str() == Token::VALUES.name->c_str())
      {
        return overwrite ? SQLType::INSERT_OVERWRITE_VALUES : SQLType::INSERT_INTO_VALUES;
      }
      else if (token_->name->c_str() == Token::ERROR.name->c_str() ||
               token_->name->c_str() == Token::XEOF.name->c_str())
      {
        if (overwrite)
        {
          sqlType = SQLType::INSERT_OVERWRITE;
        }
        break;
      }
    }
    return sqlType;
  }

  return sqlType;
}

void Lexer::nextTokenAlias()
{
  startPos = pos_;
  bufPos = 0;
  while (CharTypes::isWhitespace(ch))
  {
    if (ch == L'\n')
    {
      line++;
    }

    ch = charAt(++pos_);
    startPos = pos_;
    continue;
  }

  if (ch == L'"')
  {
    scanAlias();
  }
  else if (ch == L'\'')
  {
    scanAlias();

    int p;
    if (stringVal_.length() > 1 && stringVal_.find(L'"') == std::wstring::npos && ((p = stringVal_.find(L'\'', 1)) == std::wstring::npos || p == stringVal_.length() - 1))
    {
      std::wstring tmp = stringVal_;
      wchar_t *chars = &tmp[0];
      chars[0] = L'"';
      chars[wcslen(chars) - 1] = L'"';
      stringVal_ = chars;
    }
    token_ = Token_ptr(new Token(&Token::LITERAL_ALIAS));
  }
  else
  {
    nextToken();
  }
}

void Lexer::nextPath()
{
  while (CharTypes::isWhitespace(ch))
  {
    ch = charAt(++pos_);
  }
  stringVal_ = nullptr;
  mark_ = pos_;
  while (!CharTypes::isWhitespace(ch) && ch != L';' && pos_ < text.length())
  {
    ch = charAt(++pos_);
  }
  bufPos = pos_ - mark_;

  if (ch != L';')
  {
    ch = charAt(++pos_);
  }

  token_ = Token_ptr(new Token(&Token::LITERAL_PATH));
}

void Lexer::nextTokenForSet()
{
  while (CharTypes::isWhitespace(ch))
  {
    ch = charAt(++pos_);
  }

  wchar_t first = ch;
  if (first == L'$')
  {
    scanVariable();
    return;
  }

  if (CharTypes::isFirstIdentifierChar(first) || (first >= L'0' && first <= L'9') || first == L'{')
  {
    stringVal_ = nullptr;
    mark_ = pos_;
    while (ch != L';' && ch != LayoutCharacters::EOI)
    {
      ch = charAt(++pos_);
    }

    bufPos = pos_ - mark_;

    stringVal_ = this->subString(mark_, bufPos);
    token_ = Token_ptr(new Token(&Token::IDENTIFIER));
    return;
  }

  nextToken();
}

BOOL_ptr Lexer::skipToNextLine(int startPosition)
{
  for (int i = 0;; ++i)
  {
    int pos_ = startPosition + i;
    wchar_t ch = charAt(pos_);
    if (ch == '\n')
    {
      this->pos_ = pos_;
      this->ch = charAt(this->pos_);
      return BOOL::TRUE;
    }

    if (ch == LayoutCharacters::EOI)
    {
      this->pos_ = pos_;
      break;
    }
  }

  return BOOL::FALSE;
}

BOOL_ptr Lexer::skipToNextLineOrParameter(int startPosition)
{
  for (int i = 0;; ++i)
  {
    int pos_ = startPosition + i;
    wchar_t ch = charAt(pos_);
    if (ch == L'\n')
    {
      this->pos_ = pos_;
      this->ch = charAt(this->pos_);
      return BOOL::TRUE;
    }
    if (ch == L'$' && charAt(pos_ + 1) == L'{')
    {
      this->pos_ = pos_;
      this->ch = charAt(this->pos_);
      return BOOL::TRUE;
    }

    if (ch == LayoutCharacters::EOI)
    {
      this->pos_ = pos_;
      break;
    }
  }

  return BOOL::FALSE;
}

void Lexer::nextToken()
{
  // LOG_INFO << "nextToken";
  startPos = pos_;
  bufPos = 0;
  if (comments != nullptr && comments->size() > 0)
  {
    comments = nullptr;
  }

  // WLOG_INFO << L"nextToken";
  this->lines = 0;
  int startLine = line;

  for (;;)
  {
    WLOG_INFO << L"ch:[" << ch << L"][" << (int)ch << L"]";
    WLOG_INFO << L"CharTypes::isWhitespace(" << ch << L")[" << CharTypes::isWhitespace(ch) << L"]";
    if (CharTypes::isWhitespace(ch))
    {
      if (ch == L'\n')
      {
        line++;

        lines = line - startLine;
      }

      ch = charAt(++pos_);
      startPos = pos_;
      continue;
    }

    WLOG_INFO << L"LayoutCharacters::EOI[" << (int)LayoutCharacters::EOI << L"]pos_:" << pos_ << ",text.length():" << text.length();
    if ((int)ch == LayoutCharacters::EOI && pos_ < text.length())
    {
      ch = charAt(++pos_);
      continue;
    }

    if (ch == L'$' && isVaraintChar(charAt(pos_ + 1)))
    {
      scanVariable();
      return;
    }

    if (CharTypes::isFirstIdentifierChar(ch))
    {
      if (ch == L'（')
      {
        scanChar();
        token_ = Token_ptr(new Token(&Token::LPAREN));
        return;
      }
      else if (ch == L'）')
      {
        scanChar();
        token_ = Token_ptr(new Token(&Token::RPAREN));
        return;
      }

      if (ch == L'N' || ch == L'n')
      {
        if (charAt(pos_ + 1) == '\'')
        {
          ++pos_;
          ch = '\'';
          scanString();
          token_ = Token_ptr(new Token(&Token::LITERAL_NCHARS));
          return;
        }
      }

      if (ch == L'—' && charAt(pos_ + 1) == L'—' && charAt(pos_ + 2) == L'\n')
      {
        pos_ += 3;
        ch = charAt(pos_);
        continue;
      }

      scanIdentifier();
      return;
    }

    if (ch == L'0')
    {
      if (charAt(pos_ + 1) == L'x')
      {
        scanChar();
        scanChar();
        scanHexaDecimal();
      }
      else
      {
        scanNumber();
      }
      return;
    }
    else if (ch == L'1' ||
             ch == L'2' ||
             ch == L'3' ||
             ch == L'4' ||
             ch == L'5' ||
             ch == L'6' ||
             ch == L'7' ||
             ch == L'8' ||
             ch == L'9')
    {
      scanNumber();
      return;
    }
    else if (ch == L',' ||
             ch == L'，')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::COMMA));
      return;
    }
    else if (ch == L'(' ||
             ch == L'（')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::LPAREN));
      return;
    }
    else if (ch == L')' ||
             ch == L'）')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::RPAREN));
      return;
    }
    else if (ch == L'[')
    {
      scanLBracket();
      return;
    }
    else if (ch == L']')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::RBRACKET));
      return;
    }
    else if (ch == L'{')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::LBRACE));
      return;
    }
    else if (ch == L'}')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::RBRACE));
      return;
    }
    else if (ch == L':')
    {
      scanChar();
      if (ch == L'=')
      {
        scanChar();
        token_ = Token_ptr(new Token(&Token::COLONEQ));
      }
      else if (ch == L':')
      {
        scanChar();
        token_ = Token_ptr(new Token(&Token::COLONCOLON));
      }
      else
      {
        if (isEnabled(SQLParserFeature_ptr(new SQLParserFeature(&SQLParserFeature::TDDLHint))) ||
            dbType->name->c_str() == DbType::hive.name->c_str() ||
            dbType->name->c_str() == DbType::odps.name->c_str())
        {
          token_ = Token_ptr(new Token(&Token::COLON));
          return;
        }
        unscan();
        scanVariable();
      }
      return;
    }
    else if (ch == L'#')
    {
      scanSharp();
      if ((token_->name->c_str() == Token::LINE_COMMENT.name->c_str() ||
           token_->name->c_str() == Token::MULTI_LINE_COMMENT.name->c_str()) &&
          skipComment)
      {
        bufPos = 0;
        continue;
      }
      return;
    }
    else if (ch == L'.')
    {
      scanChar();
      if (isDigit(ch) &&
          (pos_ == 1 ||
           token_->name->c_str() != Token::IDENTIFIER.name->c_str()))
      {
        unscan();
        scanNumber();
        return;
      }
      else if (ch == L'.')
      {
        scanChar();
        if (ch == L'.')
        {
          scanChar();
          token_ = Token_ptr(new Token(&Token::DOTDOTDOT));
        }
        else
        {
          token_ = Token_ptr(new Token(&Token::DOTDOT));
        }
      }
      else
      {
        token_ = Token_ptr(new Token(&Token::DOT));
      }
      return;
    }
    else if (ch == L'\'')
    {
      scanString();
      return;
    }
    else if (ch == L'\"')
    {
      scanAlias();
      return;
    }
    else if (ch == L'*')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::STAR));
      return;
    }
    else if (ch == L'?')
    {
      scanChar();
      if (ch == L'?' &&
          DbType::postgresql.name->c_str() == dbType->name->c_str())
      {
        scanChar();
        if (ch == L'|')
        {
          scanChar();
          token_ = Token_ptr(new Token(&Token::QUESBAR));
        }
        else
        {
          token_ = Token_ptr(new Token(&Token::QUESQUES));
        }
      }
      else if (ch == L'|' &&
               DbType::postgresql.name->c_str() == dbType->name->c_str())
      {
        scanChar();
        if (ch == L'|')
        {
          unscan();
          token_ = Token_ptr(new Token(&Token::QUES));
        }
        else
        {
          token_ = Token_ptr(new Token(&Token::QUESBAR));
        }
      }
      else if (ch == L'&' &&
               DbType::postgresql.name->c_str() == dbType->name->c_str())
      {
        scanChar();
        token_ = Token_ptr(new Token(&Token::QUESAMP));
      }
      else
      {
        token_ = Token_ptr(new Token(&Token::QUES));
      }
      return;
    }
    else if (ch == L';')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::SEMI));
      return;
    }
    else if (ch == L'`')
    {
      std::string tmp = "TODO. ";
      tmp += encode_util::UnicodeToUTF8(info());
      throw new ParserException(make_string_ptr(tmp)); // TODO
    }
    else if (ch == L'@')
    {
      scanVariable_at();
      return;
    }
    else if (ch == L'-')
    {
      wchar_t next = charAt(pos_ + 1);
      if (next == L'-')
      {
        scanComment();
        if ((token_->name->c_str() == Token::LINE_COMMENT.name->c_str() ||
             token_->name->c_str() == Token::MULTI_LINE_COMMENT.name->c_str()) &&
            skipComment)
        {
          bufPos = 0;
          continue;
        }
      }
      else if (next >= L'0' && next <= L'9')
      {
        if (token_ == nullptr)
        {
          scanNumber();
          return;
        }
        if (token_->name->c_str() == Token::COMMA.name->c_str() ||
            token_->name->c_str() == Token::LPAREN.name->c_str() ||
            token_->name->c_str() == Token::WITH.name->c_str() ||
            token_->name->c_str() == Token::BY.name->c_str())
        {
          scanNumber();
        }
        else
        {
          scanOperator();
        }
      }
      else
      {
        scanOperator();
      }
      return;
    }
    else if (ch == L'/')
    {
      wchar_t nextChar = charAt(pos_ + 1);
      if (nextChar == L'/' || nextChar == L'*' || (nextChar == L'!' && isEnabled(SQLParserFeature_ptr(new SQLParserFeature(&SQLParserFeature::TDDLHint)))))
      {
        scanComment();
        if ((token_->name->c_str() == Token::LINE_COMMENT.name->c_str() ||
             token_->name->c_str() == Token::MULTI_LINE_COMMENT.name->c_str()) &&
            skipComment)
        {
          bufPos = 0;
          continue;
        }
      }
      else if (nextChar == L' ' && charAt(pos_ + 2) == L'*')
      {
        scanComment();
        if ((token_->name->c_str() == Token::LINE_COMMENT.name->c_str() ||
             token_->name->c_str() == Token::MULTI_LINE_COMMENT.name->c_str()) &&
            skipComment)
        {
          bufPos = 0;
          continue;
        }
      }
      else if (nextChar == L' ' && charAt(pos_ + 2) == L' ' && charAt(pos_ + 3) == L'*')
      {
        scanComment();
        if ((token_->name->c_str() == Token::LINE_COMMENT.name->c_str() ||
             token_->name->c_str() == Token::MULTI_LINE_COMMENT.name->c_str()) &&
            skipComment)
        {
          bufPos = 0;
          continue;
        }
      }
      else
      {
        token_ = Token_ptr(new Token(&Token::SLASH));
        scanChar();
      }
      return;
    }
    else
    {
      if (encode_util::isalpha(ch))
      {
        LOG_INFO << "goto scanIdentifier";
        scanIdentifier();
        return;
      }

      if (isOperator(ch))
      {
        LOG_INFO << "goto scanOperator";
        scanOperator();
        return;
      }

      if (ch == L'\\' && charAt(pos_ + 1) == L'N' && DbType::mysql.name->c_str() == dbType->name->c_str())
      {
        scanChar();
        scanChar();
        token_ = Token_ptr(new Token(&Token::XNULL));
        return;
      }

      // QS_TODO ?
      if (isEOF())
      { // JLS
        token_ = Token_ptr(new Token(&Token::XEOF));
      }
      else
      {
        Object_list_ptr tmp = std::make_shared<std::list<Object_ptr>>();
        Object_ptr tmpObject = Object_ptr(new Object());
        tmpObject->any_data_ = ch;
        lexError(L"illegal.wchar_t", tmp);
        scanChar();
      }

      return;
    }
  }
}

void Lexer::scanLBracket()
{
  scanChar();
  token_ = Token_ptr(new Token(&Token::LBRACKET));
}

void Lexer::scanOperator()
{
  WLOG_DEBUG << L"ch:" << ch;
  if (ch == L'+')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::PLUS));
  }
  else if (ch == L'-')
  {
    scanChar();
    if (ch == '>')
    {
      scanChar();
      if (ch == '>')
      {
        scanChar();
        token_ = Token_ptr(new Token(&Token::SUBGTGT));
      }
      else
      {
        token_ = Token_ptr(new Token(&Token::SUBGT));
      }
    }
    else
    {
      token_ = Token_ptr(new Token(&Token::SUB));
    }
  }
  else if (ch == L'*')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::STAR));
  }
  else if (ch == L'/')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::SLASH));
  }
  else if (ch == L'&')
  {
    scanChar();
    if (ch == '&')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::AMPAMP));
    }
    else
    {
      token_ = Token_ptr(new Token(&Token::AMP));
    }
  }
  else if (ch == L'|')
  {
    scanChar();
    if (ch == '|')
    {
      scanChar();
      if (ch == '/')
      {
        scanChar();
        token_ = Token_ptr(new Token(&Token::BARBARSLASH));
      }
      else
      {
        token_ = Token_ptr(new Token(&Token::BARBAR));
      }
    }
    else if (ch == '/')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::BARSLASH));
    }
    else
    {
      token_ = Token_ptr(new Token(&Token::BAR));
    }
  }
  else if (ch == L'^')
  {
    scanChar();
    if (ch == '=')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::CARETEQ));
    }
    else
    {
      token_ = Token_ptr(new Token(&Token::CARET));
    }
  }
  else if (ch == L'%')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::PERCENT));
  }
  else if (ch == L'=')
  {
    scanChar();
    if (ch == ' ')
    {
      scanChar();
    }

    if (ch == '=')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::EQEQ));
    }
    else if (ch == '>')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::EQGT));
    }
    else
    {
      token_ = Token_ptr(new Token(&Token::EQ));
    }
  }
  else if (ch == L'>')
  {
    scanChar();
    if (ch == '=')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::GTEQ));
    }
    else if (ch == '>')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::GTGT));
    }
    else
    {
      token_ = Token_ptr(new Token(&Token::GT));
    }
  }
  else if (ch == L'<')
  {
    scanChar();
    if (ch == '=')
    {
      scanChar();
      if (ch == '>')
      {
        token_ = Token_ptr(new Token(&Token::LTEQGT));
        scanChar();
      }
      else
      {
        token_ = Token_ptr(new Token(&Token::LTEQ));
      }
    }
    else if (ch == '>')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::LTGT));
    }
    else if (ch == '<')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::LTLT));
    }
    else if (ch == '@')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::LT_MONKEYS_AT));
    }
    else if (ch == '-' && charAt(pos_ + 1) == '>')
    {
      scanChar();
      scanChar();
      token_ = Token_ptr(new Token(&Token::LT_SUB_GT));
    }
    else
    {
      if (ch == ' ')
      {
        wchar_t c1 = charAt(pos_ + 1);
        if (c1 == '=')
        {
          scanChar();
          scanChar();
          if (ch == '>')
          {
            token_ = Token_ptr(new Token(&Token::LTEQGT));
            scanChar();
          }
          else
          {
            token_ = Token_ptr(new Token(&Token::LTEQ));
          }
        }
        else if (c1 == '>')
        {
          scanChar();
          scanChar();
          token_ = Token_ptr(new Token(&Token::LTGT));
        }
        else if (c1 == '<')
        {
          scanChar();
          scanChar();
          token_ = Token_ptr(new Token(&Token::LTLT));
        }
        else if (c1 == '@')
        {
          scanChar();
          scanChar();
          token_ = Token_ptr(new Token(&Token::LT_MONKEYS_AT));
        }
        else if (c1 == '-' && charAt(pos_ + 2) == '>')
        {
          scanChar();
          scanChar();
          scanChar();
          token_ = Token_ptr(new Token(&Token::LT_SUB_GT));
        }
        else
        {
          token_ = Token_ptr(new Token(&Token::LT));
        }
      }
      else
      {
        token_ = Token_ptr(new Token(&Token::LT));
      }
    }
  }
  else if (ch == L'!')
  {
    scanChar();
    while (CharTypes::isWhitespace(ch))
    {
      scanChar();
    }
    if (ch == '=')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::BANGEQ));
    }
    else if (ch == '>')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::BANGGT));
    }
    else if (ch == '<')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::BANGLT));
    }
    else if (ch == '!')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::BANGBANG)); // postsql
    }
    else if (ch == '~')
    {
      scanChar();
      if (ch == '*')
      {
        scanChar();
        token_ = Token_ptr(new Token(&Token::BANG_TILDE_STAR)); // postsql
      }
      else
      {
        token_ = Token_ptr(new Token(&Token::BANG_TILDE)); // postsql
      }
    }
    else
    {
      token_ = Token_ptr(new Token(&Token::BANG));
    }
  }
  else if (ch == L'?')
  {
    scanChar();
    token_ = Token_ptr(new Token(&Token::QUES));
  }
  else if (ch == L'~')
  {
    scanChar();
    if (ch == '*')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::TILDE_STAR));
    }
    else if (ch == '=')
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::TILDE_EQ)); // postsql
    }
    else
    {
      token_ = Token_ptr(new Token(&Token::TILDE));
    }
  }
  else
  {
    std::string tmp = "TODO. ";
    tmp += encode_util::UnicodeToUTF8(info());
    throw new ParserException(make_string_ptr(tmp));
  }
}

void Lexer::scanString()
{
  mark_ = pos_;
  bool hasSpecial = false;
  Token_ptr preToken = this->token_;

  for (;;)
  {
    if (isEOF())
    {
      LOG_DEBUG << "isEOF";
      Object_list_ptr tmp = std::make_shared<std::list<Object_ptr>>();
      lexError(L"unclosed.str.lit", tmp);
      return;
    }

    ch = charAt(++pos_);

    WLOG_DEBUG << L"ch:" << ch;
    if (ch == L'\'')
    {
      scanChar();
      if (ch != L'\'')
      {
        token_ = Token_ptr(new Token(&Token::LITERAL_CHARS));
        break;
      }
      else
      {
        if (!hasSpecial)
        {
          initBuff(bufPos);
          arraycopy(mark_ + 1, buf, 0, bufPos);
          hasSpecial = true;
        }
        putChar(L'\'');
        continue;
      }
    }

    if (!hasSpecial)
    {
      bufPos++;
      continue;
    }

    if (bufPos == wcslen(buf))
    {
      putChar(ch);
    }
    else
    {
      buf[bufPos++] = ch;
    }
  }

  if (!hasSpecial)
  {
    if (preToken->name->c_str() == Token::AS.name->c_str())
    {
      stringVal_ = subString(mark_, bufPos + 2);
    }
    else
    {
      stringVal_ = subString(mark_ + 1, bufPos);
    }
  }
  else
  {
    stringVal_ = buf;
  }
}

void Lexer::scanString2()
{
  {
    bool hasSpecial = false;
    int startIndex = pos_ + 1;
    int endIndex = -1; // text.indexOf('\'', startIndex);
    for (int i = startIndex; i < text.length(); ++i)
    {
      wchar_t ch = text.at(i);
      if (ch == L'\\')
      {
        hasSpecial = true;
        continue;
      }
      if (ch == L'\'')
      {
        endIndex = i;
        break;
      }
    }

    if (endIndex == -1)
    {
      std::string tmp = "unclosed str. ";
      tmp += encode_util::UnicodeToUTF8(info());
      throw new ParserException(make_string_ptr(tmp));
    }

    std::wstring stringVal_;
    if (token_->name->c_str() == Token::AS.name->c_str())
    {
      stringVal_ = text.substr(pos_, endIndex + 1);
    }
    else
    {
      if (startIndex == endIndex)
      {
        stringVal_ = L"";
      }
      else
      {
        stringVal_ = text.substr(startIndex, endIndex);
      }
    }
    // hasSpecial = stringVal_.indexOf('\\') != -1;

    if (!hasSpecial)
    {
      this->stringVal_ = stringVal_;
      int pos_ = endIndex + 1;
      wchar_t ch = charAt(pos_);
      if (ch != L'\'')
      {
        this->pos_ = pos_;
        this->ch = ch;
        token_ = Token_ptr(new Token(&Token::LITERAL_CHARS));
        return;
      }
    }
  }

  mark_ = pos_;
  bool hasSpecial = false;
  for (;;)
  {
    if (isEOF())
    {
      Object_list_ptr tmp = std::make_shared<std::list<Object_ptr>>();
      lexError(L"unclosed.str.lit", tmp);
      return;
    }

    ch = charAt(++pos_);

    WLOG_DEBUG << L"ch:" << ch;
    if (ch == L'\\')
    {
      scanChar();
      if (!hasSpecial)
      {
        initBuff(bufPos);
        arraycopy(mark_ + 1, buf, 0, bufPos);
        hasSpecial = true;
      }

      if (ch == L'0')
      {
        putChar(L'\0');
      }
      else if (ch == L'\'')
      {
        putChar(L'\'');
      }
      else if (ch == L'"')
      {
        putChar(L'"');
      }
      else if (ch == L'b')
      {
        putChar(L'\b');
      }
      else if (ch == L'n')
      {
        putChar(L'\n');
      }
      else if (ch == L'r')
      {
        putChar(L'\r');
      }
      else if (ch == L't')
      {
        putChar(L'\t');
      }
      else if (ch == L'\\')
      {
        putChar(L'\\');
      }
      else if (ch == L'Z')
      {
        putChar((wchar_t)0x1A); // ctrl + Z
      }
      else if (ch == L'%')
      {
        if (dbType->name->c_str() == DbType::mysql.name->c_str())
        {
          putChar(L'\\');
        }
        putChar(L'%');
      }
      else if (ch == L'_')
      {
        if (dbType->name->c_str() == DbType::mysql.name->c_str())
        {
          putChar(L'\\');
        }
        putChar(L'_');
      }
      else if (ch == L'u')
      {
        if ((features & SQLParserFeature::SupportUnicodeCodePoint.mask) != 0)
        {
          wchar_t c1 = charAt(++pos_);
          wchar_t c2 = charAt(++pos_);
          wchar_t c3 = charAt(++pos_);
          wchar_t c4 = charAt(++pos_);

          std::wstring tmp;
          tmp += c1;
          tmp += c2;
          tmp += c3;
          tmp += c4;
          int intVal = atoi(encode_util::UnicodeToUTF8(tmp).c_str());

          putChar((wchar_t)intVal);
        }
        else
        {
          putChar(ch);
        }
      }
      else
      {
        putChar(ch);
      }

      continue;
    }
    if (ch == L'\'')
    {
      scanChar();
      if (ch != L'\'')
      {
        token_ = Token_ptr(new Token(&Token::LITERAL_CHARS));
        break;
      }
      else
      {
        if (!hasSpecial)
        {
          initBuff(bufPos);
          arraycopy(mark_ + 1, buf, 0, bufPos);
          hasSpecial = true;
        }
        putChar(L'\'');
        continue;
      }
    }

    if (!hasSpecial)
    {
      bufPos++;
      continue;
    }

    if (bufPos == wcslen(buf))
    {
      putChar(ch);
    }
    else
    {
      buf[bufPos++] = ch;
    }
  }

  if (!hasSpecial)
  {
    stringVal_ = subString(mark_ + 1, bufPos);
  }
  else
  {
    stringVal_ = buf;
  }
}

void Lexer::scanString2_d()
{
  {
    bool hasSpecial = false;
    int startIndex = pos_ + 1;
    int endIndex = -1; // text.indexOf('\'', startIndex);
    for (int i = startIndex; i < text.length(); ++i)
    {
      wchar_t ch = text.at(i);
      if (ch == L'\\')
      {
        int i1 = i + 1;
        if (i1 < text.length())
        {
          hasSpecial = true;
          ++i;
          continue;
        }
      }

      if (ch == L'"')
      {
        int i1 = i + 1;
        if (i1 < text.length() && text.at(i1) == L'"')
        {
          hasSpecial = true;
          ++i;
          continue;
        }
        endIndex = i;
        break;
      }
    }

    if (endIndex == -1)
    {
      std::string tmp = "unclosed str. ";
      tmp += encode_util::UnicodeToUTF8(info());
      throw new ParserException(make_string_ptr(tmp));
    }

    std::wstring stringVal_;
    if (token_->name->c_str() == Token::AS.name->c_str())
    {
      stringVal_ = subString(pos_, endIndex + 1 - pos_);
    }
    else
    {
      if (charAt(endIndex + 1) == L'.')
      {
        stringVal_ = subString(startIndex - 1, (endIndex - startIndex) + 2);
      }
      else
      {
        stringVal_ = subString(startIndex, endIndex - startIndex);
      }
    }
    // hasSpecial = stringVal_.indexOf('\\') != -1;

    if (!hasSpecial)
    {
      this->stringVal_ = stringVal_;
      int pos_ = endIndex + 1;
      wchar_t ch = charAt(pos_);
      if (ch == L'.')
      {
        this->pos_ = pos_;
        this->ch = ch;
        token_ = Token_ptr(new Token(&Token::IDENTIFIER));
        return;
      }

      if (ch != L'\'')
      {
        this->pos_ = pos_;
        this->ch = ch;
        token_ = Token_ptr(new Token(&Token::LITERAL_CHARS));
        return;
      }
    }
  }

  mark_ = pos_;
  bool hasSpecial = false;
  for (;;)
  {
    if (isEOF())
    {
      Object_list_ptr tmp = std::make_shared<std::list<Object_ptr>>();
      lexError(L"unclosed.str.lit", tmp);
      return;
    }

    ch = charAt(++pos_);

    if (ch == L'\\')
    {
      scanChar();
      if (!hasSpecial)
      {
        initBuff(bufPos);
        arraycopy(mark_ + 1, buf, 0, bufPos);
        hasSpecial = true;
      }

      if (ch == L'0')
      {
        putChar(L'\0');
      }
      else if (ch == L'\'')
      {
        putChar(L'\'');
      }
      else if (ch == L'"')
      {
        putChar(L'"');
      }
      else if (ch == L'b')
      {
        putChar(L'\b');
      }
      else if (ch == L'n')
      {
        putChar(L'\n');
      }
      else if (ch == L'r')
      {
        putChar(L'\r');
      }
      else if (ch == L't')
      {
        putChar(L'\t');
      }
      else if (ch == L'\\')
      {
        putChar(L'\\');
      }
      else if (ch == L'Z')
      {
        putChar((wchar_t)0x1A); // ctrl + Z
      }
      else if (ch == L'%')
      {
        if (dbType->name->c_str() == DbType::mysql.name->c_str())
        {
          putChar(L'\\');
        }
        putChar(L'%');
      }
      else if (ch == L'_')
      {
        if (dbType->name->c_str() == DbType::mysql.name->c_str())
        {
          putChar(L'\\');
        }
        putChar(L'_');
      }
      else
      {
        putChar(ch);
      }

      continue;
    }
    if (ch == L'"')
    {
      scanChar();
      if (ch != L'"')
      {
        if (buf != nullptr && bufPos > 0)
        {
          stringVal_ = buf;
        }
        token_ = Token_ptr(new Token(&Token::LITERAL_CHARS));
        break;
      }
      else
      {
        if (!hasSpecial)
        {
          initBuff(bufPos);
          arraycopy(mark_ + 1, buf, 0, bufPos);
          hasSpecial = true;
        }
        putChar(L'"');
        continue;
      }
    }

    if (!hasSpecial)
    {
      bufPos++;
      continue;
    }

    if (bufPos == wcslen(buf))
    {
      putChar(ch);
    }
    else
    {
      buf[bufPos++] = ch;
    }
  }

  if (!hasSpecial)
  {
    stringVal_ = subString(mark_ + 1, bufPos);
  }
  else
  {
    stringVal_ = buf;
  }
}

void Lexer::scanAlias()
{
  wchar_t quote = ch;
  {
    bool hasSpecial = false;
    int startIndex = pos_ + 1;
    int endIndex = -1; // text.indexOf('\'', startIndex);
    for (int i = startIndex; i < text.length(); ++i)
    {
      wchar_t ch = text.at(i);
      if (ch == L'\\')
      {
        hasSpecial = true;
        i++;
        continue;
      }
      if (ch == quote)
      {
        if (i + 1 < text.length())
        {
          wchar_t ch_next = charAt(i + 1);
          if (ch_next == quote)
          {
            hasSpecial = true;
            i++;
            continue;
          }
        }

        endIndex = i;
        break;
      }
    }

    if (endIndex == -1)
    {
      std::string tmp = "unclosed str. ";
      tmp += encode_util::UnicodeToUTF8(info());
      throw new ParserException(make_string_ptr(tmp));
    }

    std::wstring stringVal_ = subString(pos_, endIndex + 1 - pos_);
    // hasSpecial = stringVal_.indexOf('\\') != -1;

    if (!hasSpecial)
    {
      this->stringVal_ = stringVal_;
      int pos_ = endIndex + 1;
      wchar_t ch = charAt(pos_);
      if (ch != L'\'')
      {
        this->pos_ = pos_;
        this->ch = ch;
        token_ = Token_ptr(new Token(&Token::LITERAL_ALIAS));
        return;
      }
    }
  }

  mark_ = pos_;
  initBuff(bufPos);
  // putChar(ch);

  putChar(ch);
  for (;;)
  {
    if (isEOF())
    {
      Object_list_ptr tmp = std::make_shared<std::list<Object_ptr>>();
      lexError(L"unclosed.str.lit", tmp);
      return;
    }

    ch = charAt(++pos_);

    if (ch == L'\\')
    {
      scanChar();

      if (ch == L'0')
      {
        putChar(L'\0');
      }
      else if (ch == L'\'')
      {
        if (ch == quote)
        {
          putChar(L'\\');
        }
        putChar(L'\'');
      }
      else if (ch == L'"')
      {
        if (ch == quote)
        {
          putChar(L'\\');
        }
        putChar(L'"');
      }
      else if (ch == L'b')
      {
        putChar(L'\b');
      }
      else if (ch == L'n')
      {
        putChar(L'\n');
      }
      else if (ch == L'r')
      {
        putChar(L'\r');
      }
      else if (ch == L't')
      {
        putChar(L'\t');
      }
      else if (ch == L'\\')
      {
        putChar(L'\\');
        putChar(L'\\');
      }
      else if (ch == L'Z')
      {
        putChar((wchar_t)0x1A); // ctrl + Z
      }
      else if (ch == L'u')
      {
        if (dbType->name->c_str() == DbType::hive.name->c_str())
        {
          wchar_t c1 = charAt(++pos_);
          wchar_t c2 = charAt(++pos_);
          wchar_t c3 = charAt(++pos_);
          wchar_t c4 = charAt(++pos_);

          std::wstring tmp;
          tmp += c1;
          tmp += c2;
          tmp += c3;
          tmp += c4;
          int intVal = atoi(encode_util::UnicodeToUTF8(tmp).c_str());

          putChar((wchar_t)intVal);
        }
        else
        {
          putChar(ch);
        }
      }
      else
      {
        putChar(ch);
      }

      continue;
    }

    if (ch == quote)
    {
      wchar_t ch_next = charAt(pos_ + 1);

      if (ch_next == quote)
      {
        putChar(L'\\');
        putChar(ch);
        scanChar();
        continue;
      }

      putChar(ch);
      scanChar();
      token_ = Token_ptr(new Token(&Token::LITERAL_ALIAS));
      break;
    }

    if (bufPos == wcslen(buf))
    {
      putChar(ch);
    }
    else
    {
      buf[bufPos++] = ch;
    }
  }

  stringVal_ = buf;
}

void Lexer::scanSharp()
{
  scanVariable();
}

void Lexer::scanVariable()
{
  if (ch != L':' && ch != L'#' && ch != L'$' && !(ch == L'@' && dbType->name->c_str() == DbType::odps.name->c_str()))
  {
    std::string tmp = "illegal variable. ";
    tmp += encode_util::UnicodeToUTF8(info());
    throw new ParserException(make_string_ptr(tmp));
  }

  mark_ = pos_;
  bufPos = 1;
  wchar_t ch;

  wchar_t c1 = charAt(pos_ + 1);
  if (c1 == L'>' && DbType::postgresql.name->c_str() == dbType->name->c_str())
  {
    pos_ += 2;
    token_ = Token_ptr(new Token(&Token::MONKEYS_AT_GT));
    this->ch = charAt(++pos_);
    return;
  }
  else if (c1 == L'{')
  {
    pos_++;
    bufPos++;

    BOOL_ptr ident = BOOL::FALSE;
    for (;;)
    {
      ch = charAt(++pos_);
      if (isEOF() || ch == L';' || ch == L'；' || ch == L'\r' || ch == L'\n')
      {
        pos_--;
        bufPos--;
        break;
      }

      if (ch == L'}' && !ident)
      {
        if (CharTypes::isIdentifierChar(charAt(pos_ + 1)))
        {
          bufPos++;
          ident = BOOL::TRUE;
          continue;
        }
        break;
      }

      if (ident && ch == L'$')
      {
        if (charAt(pos_ + 1) == L'{')
        {
          bufPos++;
          ident = BOOL::FALSE;
          continue;
        }
      }

      if (ident)
      {
        if (CharTypes::isWhitespace(ch))
        {
          pos_--;
          break;
        }
        else if (ch == ',' || ch == ')' || ch == '(' || ch == ';')
        {
          pos_--;
          break;
        }
      }

      bufPos++;
    }

    if (ch != '}' && !ident)
    {
      std::string tmp = "syntax error. ";
      tmp += encode_util::UnicodeToUTF8(info());
      throw new ParserException(make_string_ptr(tmp));
    }
    ++pos_;
    bufPos++;

    wchar_t endChar = ch;
    this->ch = charAt(pos_);

    if (dbType->name->c_str() == DbType::odps.name->c_str() && !CharTypes::isWhitespace(endChar))
    {
      while (CharTypes::isIdentifierChar(this->ch) && ch != L';' && ch != L'；')
      {
        ++pos_;
        bufPos++;
        this->ch = charAt(pos_);
      }
    }

    stringVal_ = addSymbol();
    token_ = ident ? Token_ptr(new Token(&Token::IDENTIFIER)) : Token_ptr(new Token(&Token::VARIANT));
    return;
  }
  else if (c1 == L'$' && charAt(pos_ + 2) == L'{')
  {
    pos_ += 2;
    bufPos += 2;

    for (;;)
    {
      ch = charAt(++pos_);

      if (ch == L'}')
      {
        break;
      }

      bufPos++;
      continue;
    }

    if (ch != L'}')
    {
      std::string tmp = "syntax error. ";
      tmp += encode_util::UnicodeToUTF8(info());
      throw new ParserException(make_string_ptr(tmp));
    }
    ++pos_;
    bufPos++;

    this->ch = charAt(pos_);

    if (dbType->name->c_str() == DbType::odps.name->c_str())
    {
      while (CharTypes::isIdentifierChar(this->ch))
      {
        ++pos_;
        bufPos++;
        this->ch = charAt(pos_);
      }
    }

    stringVal_ = addSymbol();
    token_ = Token_ptr(new Token(&Token::VARIANT));
    return;
  }

  for (;;)
  {
    ch = charAt(++pos_);

    if (!CharTypes::isIdentifierChar(ch))
    {
      break;
    }

    bufPos++;
    continue;
  }

  this->ch = charAt(pos_);

  stringVal_ = addSymbol();
  token_ = Token_ptr(new Token(&Token::VARIANT));
}

void Lexer::scanVariable_at()
{
  if (ch != L'@')
  {
    std::string tmp = "illegal variable. ";
    tmp += encode_util::UnicodeToUTF8(info());
    throw new ParserException(make_string_ptr(tmp));
  }

  mark_ = pos_;
  bufPos = 1;
  wchar_t ch;

  wchar_t c1 = charAt(pos_ + 1);
  if (c1 == L'@')
  {
    ++pos_;
    bufPos++;
  }

  for (;;)
  {
    ch = charAt(++pos_);

    if (!CharTypes::isIdentifierChar(ch))
    {
      break;
    }

    bufPos++;
    continue;
  }

  this->ch = charAt(pos_);

  stringVal_ = addSymbol();
  token_ = Token_ptr(new Token(&Token::VARIANT));
}

void Lexer::scanComment()
{
  if (!allowComment)
  {
    throw new NotAllowCommentException();
  }

  if ((ch == L'/' && charAt(pos_ + 1) == L'/') || (ch == L'-' && charAt(pos_ + 1) == L'-'))
  {
    scanSingleLineComment();
  }
  else if (ch == L'/' && charAt(pos_ + 1) == L'*')
  {
    scanMultiLineComment();
  }
  else
  {
    throw new IllegalStateException();
  }
}

void Lexer::scanHiveComment()
{
  if (ch != L'/' && ch != L'-')
  {
    throw new IllegalStateException();
  }

  Token_ptr lastToken = this->token_;

  mark_ = pos_;
  bufPos = 0;
  scanChar();

  if (ch == L' ')
  {
    mark_ = pos_;
    bufPos = 0;
    scanChar();

    if (dbType->name->c_str() == DbType::odps.name->c_str() && ch == L' ')
    {
      mark_ = pos_;
      bufPos = 0;
      scanChar();
    }
  }

  // /*+ */
  if (ch == L'*')
  {
    scanChar();
    bufPos++;

    while (ch == L' ')
    {
      scanChar();
      bufPos++;
    }

    BOOL_ptr isHint = BOOL::FALSE;
    int startHintSp = bufPos + 1;
    if (ch == L'+')
    {
      isHint = BOOL::TRUE;
      scanChar();
      bufPos++;
    }

    for (;;)
    {
      if (ch == L'*')
      {
        if (charAt(pos_ + 1) == L'/')
        {
          bufPos += 2;
          scanChar();
          scanChar();
          break;
        }
        else if (CharTypes::isWhitespace(charAt(pos_ + 1)))
        {
          int i = 2;
          for (; i < 1024 * 1024; ++i)
          {
            if (!CharTypes::isWhitespace(charAt(pos_ + i)))
            {
              break;
            }
          }
          if (charAt(pos_ + i) == '/')
          {
            bufPos += 2;
            pos_ += (i + 1);
            ch = charAt(pos_);
            break;
          }
        }
      }

      scanChar();
      if (ch == LayoutCharacters::EOI)
      {
        break;
      }
      bufPos++;
    }

    if (isHint)
    {
      stringVal_ = subString(mark_ + startHintSp, (bufPos - startHintSp) - 1);
      token_ = Token_ptr(new Token(&Token::HINT));
    }
    else
    {
      stringVal_ = subString(mark_, bufPos + 1);
      token_ = Token_ptr(new Token(&Token::MULTI_LINE_COMMENT));
      commentCount++;
      if (keepComments)
      {
        addComment(stringVal_);
      }
    }

    if (commentHandler != nullptr && commentHandler->handle(lastToken, stringVal_))
    {
      return;
    }

    if (token_->name->c_str() != Token::HINT.name->c_str() && !isAllowComment())
    {
      throw new NotAllowCommentException();
    }

    return;
  }

  if (!isAllowComment())
  {
    throw new NotAllowCommentException();
  }

  if (ch == L'/' || ch == L'-')
  {
    scanChar();
    bufPos++;

    for (;;)
    {
      if (ch == L'\r')
      {
        if (charAt(pos_ + 1) == L'\n')
        {
          line++;
          bufPos += 2;
          scanChar();
          break;
        }
        bufPos++;
        break;
      }
      else if (ch == LayoutCharacters::EOI)
      {
        if (pos_ >= text.length())
        {
          break;
        }
      }

      if (ch == L'\n')
      {
        line++;
        scanChar();
        bufPos++;
        break;
      }

      scanChar();
      bufPos++;
    }

    stringVal_ = subString(mark_, ch != LayoutCharacters::EOI ? bufPos : bufPos + 1);
    token_ = Token_ptr(new Token(&Token::LINE_COMMENT));
    commentCount++;
    if (keepComments)
    {
      addComment(stringVal_);
    }

    if (isEOF())
    {
      endOfComment = BOOL::TRUE;
    }
    else
    {
      endOfComment = BOOL::FALSE;
    }

    if (commentHandler != nullptr && commentHandler->handle(lastToken, stringVal_))
    {
      return;
    }

    return;
  }
}

wstring_list_ptr Lexer::scanLineArgument()
{
  wstring_list_ptr args = std::make_shared<std::list<wstring_ptr>>();
  while (ch == L' ')
  {
    scanChar();
  }

  int start = pos_;
  for (;;)
  {
    if (ch == L' ' || ch == L'\r' || ch == L'\n' || ch == LayoutCharacters::EOI || (ch == ';' && (pos_ >= text.length() - 1 || CharTypes::isWhitespace(text.at(pos_ + 1)))))
    {
      while (pos_ < text.length() - 1)
      {
        wchar_t c1 = text.at(pos_);
        if (c1 == L' ' || c1 == L'\r' || c1 == L'\n' || c1 == L'\t')
        {
          pos_++;
          continue;
        }
        break;
      }

      std::wstring arg_tmp = text.substr(start, pos_);
      // StringUtils::trim(arg_tmp);
      arg_tmp.erase(arg_tmp.begin(), std::find_if(arg_tmp.begin(), arg_tmp.end(), std::not1(std::ptr_fun<int, int>(std::isspace))));
      arg_tmp.erase(std::find_if(arg_tmp.rbegin(), arg_tmp.rend(), std::not1(std::ptr_fun<int, int>(std::isspace))).base(), arg_tmp.end());

      wstring_ptr arg = std::make_shared<std::wstring>(arg_tmp);
      if (arg->length() > 0)
      {
        args->push_back(arg);
      }
      if (ch == L';')
      {
        break;
      }
      scanChar();
      start = pos_ - 1;
      if (ch == L'\r' || ch == L'\n' || ch == LayoutCharacters::EOI)
      {
        break;
      }
      else
      {
        continue;
      }
    }

    scanChar();
  }
  if (ch == LayoutCharacters::EOI)
  {
    token_ = Token_ptr(new Token(&Token::XEOF));
  }
  else if (ch == L';')
  {
    token_ = Token_ptr(new Token(&Token::COMMA));
  }
  else
  {
    nextToken();
  }
  return args;
}

void Lexer::scanMultiLineComment()
{
  Token_ptr lastToken = this->token_;
  int depth = 1;

  scanChar();
  scanChar();
  mark_ = pos_;
  bufPos = 0;

  for (;;)
  {
    if (ch == L'/' && charAt(pos_ + 1) == L'*')
    {
      scanChar();
      scanChar();
      if (ch == L'!' || ch == L'+')
      {
        scanChar();
        ++depth;
      }
    }

    if (ch == L'*' && charAt(pos_ + 1) == L'/')
    {
      scanChar();
      scanChar();
      if (0 == --depth)
      {
        break;
      }
    }

    // multiline comment结束符错误
    if (ch == LayoutCharacters::EOI)
    {
      std::string tmp = "unterminated /* comment. ";
      tmp += encode_util::UnicodeToUTF8(info());
      throw new ParserException(make_string_ptr(tmp));
    }
    scanChar();
    bufPos++;
  }

  stringVal_ = subString(mark_, bufPos);
  token_ = Token_ptr(new Token(&Token::MULTI_LINE_COMMENT));
  commentCount++;
  if (keepComments)
  {
    addComment(stringVal_);
  }

  if (commentHandler != nullptr && commentHandler->handle(lastToken, stringVal_))
  {
    return;
  }

  if (!isAllowComment() && !isSafeComment(stringVal_))
  {
    throw new NotAllowCommentException();
  }
}

void Lexer::scanSingleLineComment()
{
  Token_ptr lastToken = this->token_;

  mark_ = pos_;
  bufPos = 2;

  scanChar();
  scanChar();

  for (;;)
  {
    if (ch == L'\r')
    {
      if (charAt(pos_ + 1) == L'\n')
      {
        line++;
        scanChar();
        break;
      }
      bufPos++;
      break;
    }

    if (ch == L'\n')
    {
      line++;
      scanChar();
      break;
    }

    if (ch == LayoutCharacters::EOI)
    {
      break;
    }

    scanChar();
    bufPos++;
  }

  stringVal_ = subString(mark_, bufPos);
  token_ = Token_ptr(new Token(&Token::LINE_COMMENT));
  commentCount++;
  if (keepComments)
  {
    addComment(stringVal_);
  }

  if (commentHandler != nullptr && commentHandler->handle(lastToken, stringVal_))
  {
    return;
  }

  if (!isAllowComment() && !isSafeComment(stringVal_))
  {
    throw new NotAllowCommentException();
  }
}

void Lexer::scanIdentifier()
{
  LOG_INFO << "in scanIdentifier";
  this->hashLCase_ = 0;
  this->hash = 0;

  wchar_t first = ch;

  if (ch == L'`')
  {
    mark_ = pos_;
    bufPos = 1;
    wchar_t ch;

    int startPos = pos_ + 1;
    int quoteIndex = text.find(L'`', startPos);
    if (quoteIndex == std::wstring::npos)
    {
      std::string tmp = "illegal identifier. ";
      tmp += encode_util::UnicodeToUTF8(info());
      throw new ParserException(make_string_ptr(tmp));
    }

    hashLCase_ = 0xcbf29ce484222325L;
    hash = 0xcbf29ce484222325L;

    for (int i = startPos; i < quoteIndex; ++i)
    {
      ch = text.at(i);

      hashLCase_ ^= ((ch >= L'A' && ch <= L'Z') ? (ch + 32) : ch);
      hashLCase_ *= 0x100000001b3L;

      hash ^= ch;
      hash *= 0x100000001b3L;
    }

    stringVal_ = MySqlLexer::quoteTable->addSymbol(text, pos_, quoteIndex + 1 - pos_, hash);
    // stringVal_ = text.substring(mark_, pos_);
    pos_ = quoteIndex + 1;
    this->ch = charAt(pos_);
    token_ = Token_ptr(new Token(&Token::IDENTIFIER));
    return;
  }

  bool firstFlag = CharTypes::isFirstIdentifierChar(first);
  if (!firstFlag)
  {
    std::string tmp = "illegal identifier. ";
    tmp += encode_util::UnicodeToUTF8(info());
    throw new ParserException(make_string_ptr(tmp));
  }

  hashLCase_ = 0xcbf29ce484222325L;
  hash = 0xcbf29ce484222325L;

  hashLCase_ ^= ((ch >= L'A' && ch <= L'Z') ? (ch + 32) : ch);
  hashLCase_ *= 0x100000001b3L;

  hash ^= ch;
  hash *= 0x100000001b3L;

  mark_ = pos_;
  bufPos = 1;
  wchar_t ch = 0;
  for (;;)
  {
    wchar_t c0 = ch;
    ch = charAt(++pos_);

    if (!CharTypes::isIdentifierChar(ch))
    {
      if ((ch == L'（' || ch == L'）') && c0 > 256)
      {
        bufPos++;
        continue;
      }
      break;
    }

    hashLCase_ ^= ((ch >= L'A' && ch <= L'Z') ? (ch + 32) : ch);
    hashLCase_ *= 0x100000001b3L;

    hash ^= ch;
    hash *= 0x100000001b3L;

    bufPos++;
    continue;
  }

  this->ch = charAt(pos_);

  if (bufPos == 1)
  {

    if (first == L'（')
    {
      token_ = Token_ptr(new Token(&Token::LPAREN));
      return;
    }
    else if (first == L'）')
    {
      token_ = Token_ptr(new Token(&Token::RPAREN));
      return;
    }
    token_ = Token_ptr(new Token(&Token::IDENTIFIER));
    stringVal_ = CharTypes::valueOf(first);
    // if (stringVal_ == nullptr) {
    //     stringVal_ = Character.toString(first);
    // }
    return;
  }

  Token_ptr tok = keywords->getKeyword(hashLCase_);
  if (tok != nullptr)
  {
    token_ = tok;
    if (token_->name->c_str() == Token::IDENTIFIER.name->c_str())
    {
      stringVal_ = SymbolTable::global->addSymbol(text, mark_, bufPos, hash);
    }
    else
    {
      stringVal_.clear();
    }
  }
  else
  {
    token_ = Token_ptr(new Token(&Token::IDENTIFIER));
    stringVal_ = SymbolTable::global->addSymbol(text, mark_, bufPos, hash);
  }
}

void Lexer::scanNumber()
{
  mark_ = pos_;
  numberSale = 0;
  numberExp = BOOL::FALSE;
  bufPos = 0;

  if (ch == L'0' &&
      charAt(pos_ + 1) == L'b' &&
      dbType->name->c_str() != DbType::odps.name->c_str())
  {
    int i = 2;
    int mark_ = pos_ + 2;
    for (;; ++i)
    {
      wchar_t ch = charAt(pos_ + i);
      if (ch == L'0' || ch == L'1')
      {
        continue;
      }
      else if (ch >= L'2' && ch <= L'9')
      {
        break;
      }
      else
      {
        bufPos += i;
        pos_ += i;
        stringVal_ = subString(mark_, i - 2);
        this->ch = charAt(pos_);
        token_ = Token_ptr(new Token(&Token::BITS));
        return;
      }
    }
  }

  if (ch == L'-')
  {
    bufPos++;
    ch = charAt(++pos_);
  }

  for (;;)
  {
    if (ch >= L'0' && ch <= L'9')
    {
      bufPos++;
    }
    else
    {
      break;
    }
    ch = charAt(++pos_);
  }

  if (ch == L'.')
  {
    if (charAt(pos_ + 1) == L'.')
    {
      token_ = Token_ptr(new Token(&Token::LITERAL_INT));
      return;
    }
    bufPos++;
    ch = charAt(++pos_);

    for (numberSale = 0;; numberSale++)
    {
      if (ch >= L'0' && ch <= L'9')
      {
        bufPos++;
      }
      else
      {
        break;
      }
      ch = charAt(++pos_);
    }

    numberExp = BOOL::TRUE;
  }

  if ((ch == L'e' || ch == L'E') && (isDigit(charAt(pos_ + 1)) ||
                                     (isDigit2(charAt(pos_ + 1)) &&
                                      isDigit2(charAt(pos_ + 2)))))
  {
    numberExp = BOOL::TRUE;

    bufPos++;
    ch = charAt(++pos_);

    if (ch == L'+' || ch == L'-')
    {
      bufPos++;
      ch = charAt(++pos_);
    }

    for (;;)
    {
      if (ch >= L'0' && ch <= L'9')
      {
        bufPos++;
      }
      else
      {
        break;
      }
      ch = charAt(++pos_);
    }

    if ((ch >= L'a' && ch <= L'z') || (ch >= L'A' && ch <= L'Z'))
    {
      numberExp = BOOL::FALSE;
    }
  }

  if (numberSale > 0 || numberExp)
  {
    if (text.at(mark_) == L'.' && CharTypes::isIdentifierChar(ch))
    {
      pos_ = mark_ + 1;
      ch = charAt(pos_);
      token_ = Token_ptr(new Token(&Token::DOT));
      return;
    }
    token_ = Token_ptr(new Token(&Token::LITERAL_FLOAT));
    return;
  }

  if (ch != L'`')
  {
    if (CharTypes::isFirstIdentifierChar(ch) &&
        ch != L'）' &&
        !(ch == L'b' &&
          bufPos == 1 &&
          charAt(pos_ - 1) == L'0' &&
          dbType->name->c_str() != DbType::odps.name->c_str()))
    {
      bufPos++;
      BOOL_ptr brace = BOOL::FALSE;
      for (;;)
      {
        wchar_t c0 = ch;
        ch = charAt(++pos_);

        if (isEOF())
        {
          break;
        }

        if (!CharTypes::isIdentifierChar(ch))
        {
          if (ch == L'{' && charAt(pos_ - 1) == L'$' && !brace)
          {
            bufPos++;
            brace = BOOL::TRUE;
            continue;
          }

          if (ch == L'}' && brace)
          {
            bufPos++;
            brace = BOOL::FALSE;
            continue;
          }

          if ((ch == L'（' || ch == L'）') && c0 > 256)
          {
            bufPos++;
            continue;
          }
          break;
        }

        bufPos++;
        continue;
      }

      stringVal_ = addSymbol();
      hashLCase_ = FnvHash::hashCode64(make_string_ptr(encode_util::UnicodeToUTF8(stringVal_)));
      token_ = Token_ptr(new Token(&Token::IDENTIFIER));
      return;
    }
  }

  token_ = Token_ptr(new Token(&Token::LITERAL_INT));
}

void Lexer::scanHexaDecimal()
{
  mark_ = pos_;
  bufPos = 0;

  if (ch == L'-')
  {
    bufPos++;
    ch = charAt(++pos_);
  }

  for (;;)
  {
    if (CharTypes::isHex(ch))
    {
      bufPos++;
    }
    else
    {
      break;
    }
    ch = charAt(++pos_);
  }

  if (CharTypes::isIdentifierChar(ch))
  {
    for (;;)
    {
      bufPos++;
      ch = charAt(++pos_);
      if (!CharTypes::isIdentifierChar(ch))
      {
        break;
      }
    }
    mark_ -= 2;
    bufPos += 2;
    stringVal_ = addSymbol();
    hashLCase_ = FnvHash::hashCode64(make_string_ptr(encode_util::UnicodeToUTF8(stringVal_)));
    token_ = Token_ptr(new Token(&Token::IDENTIFIER));
    return;
  }

  token_ = Token_ptr(new Token(&Token::LITERAL_HEX));
}

std::wstring Lexer::hexString()
{
  return subString(mark_, bufPos);
}

bool Lexer::isDigit(wchar_t ch)
{
  return ch >= L'0' && ch <= L'9';
}

bool Lexer::isDigit2(wchar_t ch)
{
  return ch == L'+' || ch == L'-' || (ch >= L'0' && ch <= L'9');
}

/**
 * Append a character to sbuf.
 */
void Lexer::putChar(wchar_t ch)
{
  // if (bufPos == wcslen(buf)) {
  //     wchar_t[] newsbuf = new wchar_t[buf.length * 2];
  //     System.arraycopy(buf, 0, newsbuf, 0, buf.length);
  //     buf = newsbuf;
  // }
  bufPos++;
  buf += ch;
}

/**
 * Return the current token_'s position: a 0-based offset from beginning of the raw input stream (before unicode
 * translation)
 */
int Lexer::pos()
{
  return pos_;
}

/**
 * The value of a literal token_, recorded as a string. For integers, leading 0x and 'l' suffixes are suppressed.
 */
wstring_ptr Lexer::stringVal()
{
  // if (stringVal_ == nullptr) {
  //     stringVal_ = subString(mark_, bufPos);
  // }
  return make_wstring_ptr(stringVal_);
}

void Lexer::stringVal(std::stringstream out)
{
  // if (stringVal_ != nullptr) {
  //     out.append(stringVal_);
  //     return;
  // }

  out << subString(mark_, bufPos).c_str();
}

bool Lexer::identifierEquals(string_ptr text)
{
  return identifierEquals(encode_util::UTF8ToUnicode(text->c_str()));
}

bool Lexer::identifierEquals(std::wstring text)
{
  if (token_->name->c_str() != Token::IDENTIFIER.name->c_str())
  {
    return false;
  }

  if (stringVal_.empty())
  {
    stringVal_ = subString(mark_, bufPos);
  }
  return (StringUtils::equalsIgnoreCase(text, stringVal_));
}

bool Lexer::identifierEquals(long hash_lower)
{
  if (token_->name->c_str() != Token::IDENTIFIER.name->c_str())
  {
    return false;
  }

  if (this->hashLCase_ == 0)
  {
    if (stringVal_.empty())
    {
      stringVal_ = subString(mark_, bufPos);
    }
    this->hashLCase_ = FnvHash::fnv1a_64_lower(make_string_ptr(encode_util::UnicodeToUTF8(stringVal_)));
  }
  return (this->hashLCase_ == hash_lower);
}

long Lexer::hashLCase()
{
  if (this->hashLCase_ == 0)
  {
    if (stringVal_.empty())
    {
      stringVal_ = subString(mark_, bufPos);
    }
    WLOG_DEBUG << L"stringVal_:" << stringVal_;
    this->hashLCase_ = FnvHash::fnv1a_64_lower(make_string_ptr(encode_util::UnicodeToUTF8(stringVal_)));
  }
  return hashLCase_;
}

wstring_list_ptr Lexer::readAndResetComments()
{
  wstring_list_ptr comments = this->comments;

  this->comments = nullptr;

  return comments;
}

bool Lexer::isOperator(wchar_t ch)
{
  if (ch == L'!' ||
      ch == L'%' ||
      ch == L'&' ||
      ch == L'*' ||
      ch == L'+' ||
      ch == L'-' ||
      ch == L'<' ||
      ch == L'=' ||
      ch == L'>' ||
      ch == L'^' ||
      ch == L'|' ||
      ch == L'~' ||
      ch == L';')
  {
    return true;
  }
  else
  {
    return false;
  }
}

BOOL_ptr Lexer::isNegativeIntegerValue()
{
  if (charAt(mark_) == L'-')
  {
    return BOOL::TRUE;
  }
  else
  {
    return BOOL::FALSE;
  }
}

// QS_TODO negative number is invisible for lexer
long Lexer::integerValue()
{
  long result = 0;
  BOOL_ptr negative = BOOL::FALSE;
  int i = mark_, max = mark_ + bufPos;
  long limit;
  long multmin;
  int digit;

  if (charAt(mark_) == L'-')
  {
    negative = BOOL::TRUE;
    limit = LONG_MIN;
    i++;
  }
  else
  {
    limit = -LONG_MAX;
  }
  multmin = negative ? MULTMIN_RADIX_TEN : N_MULTMAX_RADIX_TEN;
  if (i < max)
  {
    digit = charAt(i++) - L'0';
    result = -digit;
  }
  while (i < max)
  {
    // Accumulating negatively avoids surprises near MAX_VALUE
    digit = charAt(i++) - L'0';
    if (result < multmin)
    {
      return atol(encode_util::UnicodeToUTF8(numberString()).c_str());
    }
    result *= 10;
    if (result < limit + digit)
    {
      return atol(encode_util::UnicodeToUTF8(numberString()).c_str());
    }
    result -= digit;
  }

  if (negative)
  {
    if (i > mark_ + 1)
    {
      if (result >= INT_MIN)
      {
        return (int)result;
      }
      return result;
    }
    else
    { /* Only got "-" */
      throw new NumberFormatException(make_string_ptr(encode_util::UnicodeToUTF8(numberString())));
    }
  }
  else
  {
    result = -result;
    if (result <= INT_MAX)
    {
      return (int)result;
    }
    return result;
  }
}

int Lexer::bp()
{
  return this->pos_;
}

wchar_t Lexer::current()
{
  return this->ch;
}

// todo fix reset reset字段会导致lexer的游标不对齐 不建议使用
//  @Deprecated
void Lexer::reset(int mark_, wchar_t markChar, Token_ptr token_)
{
  this->pos_ = mark_;
  this->ch = markChar;
  this->token_ = token_;
}

std::wstring Lexer::numberString()
{
  return text.substr(mark_, bufPos);
  // return subString(mark_, bufPos);
}

long Lexer::decimalValue()
{
  if (numberSale > 0 && !numberExp)
  {
    int len = bufPos;

    if (len < 20)
    {
      long unscaleVal = 0;

      BOOL_ptr negative = BOOL::FALSE;
      int i = 0;
      wchar_t first = text.at(mark_);
      if (first == L'+')
      {
        i++;
      }
      else if (first == L'-')
      {
        i++;
        negative = BOOL::TRUE;
      }
      for (; i < len; ++i)
      {
        wchar_t ch = text.at(mark_ + i);
        if (ch == L'.')
        {
          continue;
        }
        int digit = ch - L'0';
        unscaleVal = unscaleVal * 10 + digit;
      }
      return negative ? -unscaleVal : unscaleVal, numberSale;
    }
  }

  wstring_ptr value = sub_chars(mark_, bufPos);
  if (!StringUtils::isNumber(make_string_ptr(encode_util::UnicodeToUTF8(value->c_str()))))
  {
    std::string tmp = encode_util::UnicodeToUTF8(value->c_str());
    tmp += " is not a number! ";
    tmp += encode_util::UnicodeToUTF8(info());
    throw new ParserException(make_string_ptr(tmp));
  }
  return atol(encode_util::UnicodeToUTF8(value->c_str()).c_str());
}

SQLNumberExpr_ptr Lexer::numberExpr()
{
  wstring_ptr value = sub_chars(mark_, bufPos);
  if (!StringUtils::isNumber(make_string_ptr(encode_util::UnicodeToUTF8(value->c_str()))))
  {
    std::string tmp = encode_util::UnicodeToUTF8(value->c_str());
    tmp += " is not a number! ";
    tmp += encode_util::UnicodeToUTF8(info());
    throw new ParserException(make_string_ptr(tmp));
  }

  return SQLNumberExpr_ptr(new SQLNumberExpr(const_cast<char *>(encode_util::UnicodeToUTF8(value->c_str()).c_str())));
}

SQLNumberExpr_ptr Lexer::numberExpr(SQLObject_ptr parent)
{
  wstring_ptr value = sub_chars(mark_, bufPos);
  if (!StringUtils::isNumber(make_string_ptr(encode_util::UnicodeToUTF8(value->c_str()))))
  {
    std::string tmp = encode_util::UnicodeToUTF8(value->c_str());
    tmp += " is not a number! ";
    tmp += encode_util::UnicodeToUTF8(info());
    throw new ParserException(make_string_ptr(tmp));
  }

  return SQLNumberExpr_ptr(new SQLNumberExpr(const_cast<char *>(encode_util::UnicodeToUTF8(value->c_str()).c_str()), parent));
}

SQLNumberExpr_ptr Lexer::numberExpr(BOOL_ptr negate)
{
  wstring_ptr value = sub_chars(mark_, bufPos);
  if (!StringUtils::isNumber(make_string_ptr(encode_util::UnicodeToUTF8(value->c_str()))))
  {
    std::string tmp = encode_util::UnicodeToUTF8(value->c_str());
    tmp += " is not a number! ";
    tmp += encode_util::UnicodeToUTF8(info());
    throw new ParserException(make_string_ptr(tmp));
  }

  if (negate)
  {
    std::wstring chars;
    chars += L'-';
    // System.arraycopy(value, 0, chars, 1, value.length);

    chars += value->c_str();

    return SQLNumberExpr_ptr(new SQLNumberExpr(const_cast<char *>(encode_util::UnicodeToUTF8(chars).c_str())));
  }
  else
  {
    return SQLNumberExpr_ptr(new SQLNumberExpr(const_cast<char *>(encode_util::UnicodeToUTF8(value->c_str()).c_str())));
  }
}

BOOL_ptr Lexer::hasComment()
{
  if (comments != nullptr)
  {
    return BOOL::TRUE;
  }
  else
  {
    return BOOL::FALSE;
  }
}

int Lexer::getCommentCount()
{
  return commentCount;
}

void Lexer::skipToEOF()
{
  pos_ = text.length();
  this->token_ = Token_ptr(new Token(&Token::XEOF));
}

BOOL_ptr Lexer::isEndOfComment()
{
  return endOfComment;
}

BOOL_ptr Lexer::isSafeComment(std::wstring comment)
{
  // if (comment == nullptr) {
  //     return BOOL::TRUE;
  // }
  comment = StringUtils::toLower(comment);
  if (comment.find(L"select") != std::wstring::npos    //
      || comment.find(L"delete") != std::wstring::npos //
      || comment.find(L"insert") != std::wstring::npos //
      || comment.find(L"update") != std::wstring::npos //
      || comment.find(L"into") != std::wstring::npos   //
      || comment.find(L"where") != std::wstring::npos  //
      || comment.find(L"or") != std::wstring::npos     //
      || comment.find(L"and") != std::wstring::npos    //
      || comment.find(L"union") != std::wstring::npos  //
      || comment.find(L'\'') != std::wstring::npos     //
      || comment.find(L'=') != std::wstring::npos      //
      || comment.find(L'>') != std::wstring::npos      //
      || comment.find(L'<') != std::wstring::npos      //
      || comment.find(L'&') != std::wstring::npos      //
      || comment.find(L'|') != std::wstring::npos      //
      || comment.find(L'^') != std::wstring::npos      //
  )
  {
    return BOOL::FALSE;
  }
  return BOOL::TRUE;
}

void Lexer::addComment(std::wstring comment)
{
  if (comments == nullptr)
  {
    comments = std::make_shared<std::list<wstring_ptr>>();
  }
  comments->push_back(make_wstring_ptr(comment));
}

wstring_list_ptr Lexer::getComments()
{
  return comments;
}

int Lexer::getLine()
{
  return line;
}

void Lexer::computeRowAndColumn()
{
  int line = 1;
  int column = 1;
  for (int i = 0; i < startPos; ++i)
  {
    wchar_t ch = text.at(i);
    if (ch == '\n')
    {
      column = 0;
      line++;
    }
    else
    {
      column++;
    }
  }

  this->posLine = line;
  this->posColumn = column;
}

int Lexer::getPosLine()
{
  return posLine;
}

int Lexer::getPosColumn()
{
  return posColumn;
}

void Lexer::config(SQLParserFeature_ptr feature, BOOL_ptr state)
{
  features = SQLParserFeature::config(features, feature, state);

  if (feature->name->c_str() == SQLParserFeature::OptimizedForParameterized.name->c_str())
  {
    optimizedForParameterized = state;
  }
  else if (feature->name->c_str() == SQLParserFeature::KeepComments.name->c_str())
  {
    this->keepComments = state;
  }
  else if (feature->name->c_str() == SQLParserFeature::KeepSourceLocation.name->c_str())
  {
    this->keepSourceLocation = state;
  }
  else if (feature->name->c_str() == SQLParserFeature::SkipComments.name->c_str())
  {
    this->skipComment = state;
  }
}

time_t Lexer::getTimeZone()
{
  return timeZone;
}

void Lexer::setTimeZone(time_t timeZone)
{
  this->timeZone = timeZone;
}

BOOL_ptr Lexer::isEnabled(SQLParserFeature_ptr feature)
{
  return SQLParserFeature::isEnabled(this->features, feature);
}

std::wstring Lexer::parameterize(std::wstring sql, DbType_ptr dbType)
{
  Lexer_ptr lexer = SQLParserUtils::createLexer(make_string_ptr(encode_util::UnicodeToUTF8(sql)), dbType);
  lexer->optimizedForParameterized = BOOL::TRUE; // optimized

  lexer->nextToken();

  std::stringstream sbuf;

for_:
  for (;;)
  {
    Token_ptr token_ = lexer->token_;
    if (token_->name->c_str() == Token::LITERAL_ALIAS.name->c_str() ||
        token_->name->c_str() == Token::LITERAL_FLOAT.name->c_str() ||
        token_->name->c_str() == Token::LITERAL_CHARS.name->c_str() ||
        token_->name->c_str() == Token::LITERAL_INT.name->c_str() ||
        token_->name->c_str() == Token::LITERAL_NCHARS.name->c_str() ||
        token_->name->c_str() == Token::LITERAL_HEX.name->c_str() ||
        token_->name->c_str() == Token::VARIANT.name->c_str())
    {
      if (sbuf.str().length() != 0)
      {
        sbuf << ' ';
      }
      sbuf << '?';
    }
    else if (token_->name->c_str() == Token::COMMA.name->c_str())
    {
      sbuf << ',';
    }
    else if (token_->name->c_str() == Token::EQ.name->c_str())
    {
      sbuf << '=';
    }
    else if (token_->name->c_str() == Token::XEOF.name->c_str())
    {
      goto for_;
    }
    else if (token_->name->c_str() == Token::ERROR.name->c_str())
    {
      return sql;
    }
    else if (token_->name->c_str() == Token::SELECT.name->c_str())
    {
      sbuf << "SELECT";
    }
    else if (token_->name->c_str() == Token::UPDATE.name->c_str())
    {
      sbuf << "UPDATE";
    }
    else
    {
      if (sbuf.str().length() != 0)
      {
        sbuf << ' ';
      }
      std::string tmp;
      sbuf >> tmp;
      lexer->stringVal_ = encode_util::UTF8ToUnicode(tmp);
    }

    lexer->nextToken();
  }

  std::string stmp;
  sbuf >> stmp;
  return encode_util::UTF8ToUnicode(stmp);
}

std::wstring Lexer::getSource()
{
  return text;
}
